/* * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "modules/audio_device/android/opensles_recorder.h" #include #include #include "api/array_view.h" #include "modules/audio_device/android/audio_common.h" #include "modules/audio_device/android/audio_manager.h" #include "modules/audio_device/fine_audio_buffer.h" #include "rtc_base/arraysize.h" #include "rtc_base/checks.h" #include "rtc_base/platform_thread.h" #include "rtc_base/time_utils.h" #define TAG "OpenSLESRecorder" #define ALOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__) #define ALOGD(...) __android_log_print(ANDROID_LOG_DEBUG, TAG, __VA_ARGS__) #define ALOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__) #define ALOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__) #define ALOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) #define LOG_ON_ERROR(op) \ [](SLresult err) { \ if (err != SL_RESULT_SUCCESS) { \ ALOGE("%s:%d %s failed: %s", __FILE__, __LINE__, #op, \ GetSLErrorString(err)); \ return true; \ } \ return false; \ }(op) namespace webrtc { OpenSLESRecorder::OpenSLESRecorder(AudioManager* audio_manager) : audio_manager_(audio_manager), audio_parameters_(audio_manager->GetRecordAudioParameters()), audio_device_buffer_(nullptr), initialized_(false), recording_(false), engine_(nullptr), recorder_(nullptr), simple_buffer_queue_(nullptr), buffer_index_(0), last_rec_time_(0) { ALOGD("ctor[tid=%d]", rtc::CurrentThreadId()); // Detach from this thread since we want to use the checker to verify calls // from the internal audio thread. thread_checker_opensles_.Detach(); // Use native audio output parameters provided by the audio manager and // define the PCM format structure. pcm_format_ = CreatePCMConfiguration(audio_parameters_.channels(), audio_parameters_.sample_rate(), audio_parameters_.bits_per_sample()); } OpenSLESRecorder::~OpenSLESRecorder() { ALOGD("dtor[tid=%d]", rtc::CurrentThreadId()); RTC_DCHECK(thread_checker_.IsCurrent()); Terminate(); DestroyAudioRecorder(); engine_ = nullptr; RTC_DCHECK(!engine_); RTC_DCHECK(!recorder_); RTC_DCHECK(!simple_buffer_queue_); } int OpenSLESRecorder::Init() { ALOGD("Init[tid=%d]", rtc::CurrentThreadId()); RTC_DCHECK(thread_checker_.IsCurrent()); if (audio_parameters_.channels() == 2) { ALOGD("Stereo mode is enabled"); } return 0; } int OpenSLESRecorder::Terminate() { ALOGD("Terminate[tid=%d]", rtc::CurrentThreadId()); RTC_DCHECK(thread_checker_.IsCurrent()); StopRecording(); return 0; } int OpenSLESRecorder::InitRecording() { ALOGD("InitRecording[tid=%d]", rtc::CurrentThreadId()); RTC_DCHECK(thread_checker_.IsCurrent()); RTC_DCHECK(!initialized_); RTC_DCHECK(!recording_); if (!ObtainEngineInterface()) { ALOGE("Failed to obtain SL Engine interface"); return -1; } CreateAudioRecorder(); initialized_ = true; buffer_index_ = 0; return 0; } int OpenSLESRecorder::StartRecording() { ALOGD("StartRecording[tid=%d]", rtc::CurrentThreadId()); RTC_DCHECK(thread_checker_.IsCurrent()); RTC_DCHECK(initialized_); RTC_DCHECK(!recording_); if (fine_audio_buffer_) { fine_audio_buffer_->ResetRecord(); } // Add buffers to the queue before changing state to SL_RECORDSTATE_RECORDING // to ensure that recording starts as soon as the state is modified. On some // devices, SLAndroidSimpleBufferQueue::Clear() used in Stop() does not flush // the buffers as intended and we therefore check the number of buffers // already queued first. Enqueue() can return SL_RESULT_BUFFER_INSUFFICIENT // otherwise. int num_buffers_in_queue = GetBufferCount(); for (int i = 0; i < kNumOfOpenSLESBuffers - num_buffers_in_queue; ++i) { if (!EnqueueAudioBuffer()) { recording_ = false; return -1; } } num_buffers_in_queue = GetBufferCount(); RTC_DCHECK_EQ(num_buffers_in_queue, kNumOfOpenSLESBuffers); LogBufferState(); // Start audio recording by changing the state to SL_RECORDSTATE_RECORDING. // Given that buffers are already enqueued, recording should start at once. // The macro returns -1 if recording fails to start. last_rec_time_ = rtc::Time(); if (LOG_ON_ERROR( (*recorder_)->SetRecordState(recorder_, SL_RECORDSTATE_RECORDING))) { return -1; } recording_ = (GetRecordState() == SL_RECORDSTATE_RECORDING); RTC_DCHECK(recording_); return 0; } int OpenSLESRecorder::StopRecording() { ALOGD("StopRecording[tid=%d]", rtc::CurrentThreadId()); RTC_DCHECK(thread_checker_.IsCurrent()); if (!initialized_ || !recording_) { return 0; } // Stop recording by setting the record state to SL_RECORDSTATE_STOPPED. if (LOG_ON_ERROR( (*recorder_)->SetRecordState(recorder_, SL_RECORDSTATE_STOPPED))) { return -1; } // Clear the buffer queue to get rid of old data when resuming recording. if (LOG_ON_ERROR((*simple_buffer_queue_)->Clear(simple_buffer_queue_))) { return -1; } thread_checker_opensles_.Detach(); initialized_ = false; recording_ = false; return 0; } void OpenSLESRecorder::AttachAudioBuffer(AudioDeviceBuffer* audio_buffer) { ALOGD("AttachAudioBuffer"); RTC_DCHECK(thread_checker_.IsCurrent()); RTC_CHECK(audio_buffer); audio_device_buffer_ = audio_buffer; // Ensure that the audio device buffer is informed about the native sample // rate used on the recording side. const int sample_rate_hz = audio_parameters_.sample_rate(); ALOGD("SetRecordingSampleRate(%d)", sample_rate_hz); audio_device_buffer_->SetRecordingSampleRate(sample_rate_hz); // Ensure that the audio device buffer is informed about the number of // channels preferred by the OS on the recording side. const size_t channels = audio_parameters_.channels(); ALOGD("SetRecordingChannels(%zu)", channels); audio_device_buffer_->SetRecordingChannels(channels); // Allocated memory for internal data buffers given existing audio parameters. AllocateDataBuffers(); } int OpenSLESRecorder::EnableBuiltInAEC(bool enable) { ALOGD("EnableBuiltInAEC(%d)", enable); RTC_DCHECK(thread_checker_.IsCurrent()); ALOGE("Not implemented"); return 0; } int OpenSLESRecorder::EnableBuiltInAGC(bool enable) { ALOGD("EnableBuiltInAGC(%d)", enable); RTC_DCHECK(thread_checker_.IsCurrent()); ALOGE("Not implemented"); return 0; } int OpenSLESRecorder::EnableBuiltInNS(bool enable) { ALOGD("EnableBuiltInNS(%d)", enable); RTC_DCHECK(thread_checker_.IsCurrent()); ALOGE("Not implemented"); return 0; } bool OpenSLESRecorder::ObtainEngineInterface() { ALOGD("ObtainEngineInterface"); RTC_DCHECK(thread_checker_.IsCurrent()); if (engine_) return true; // Get access to (or create if not already existing) the global OpenSL Engine // object. SLObjectItf engine_object = audio_manager_->GetOpenSLEngine(); if (engine_object == nullptr) { ALOGE("Failed to access the global OpenSL engine"); return false; } // Get the SL Engine Interface which is implicit. if (LOG_ON_ERROR( (*engine_object) ->GetInterface(engine_object, SL_IID_ENGINE, &engine_))) { return false; } return true; } bool OpenSLESRecorder::CreateAudioRecorder() { ALOGD("CreateAudioRecorder"); RTC_DCHECK(thread_checker_.IsCurrent()); if (recorder_object_.Get()) return true; RTC_DCHECK(!recorder_); RTC_DCHECK(!simple_buffer_queue_); // Audio source configuration. SLDataLocator_IODevice mic_locator = {SL_DATALOCATOR_IODEVICE, SL_IODEVICE_AUDIOINPUT, SL_DEFAULTDEVICEID_AUDIOINPUT, NULL}; SLDataSource audio_source = {&mic_locator, NULL}; // Audio sink configuration. SLDataLocator_AndroidSimpleBufferQueue buffer_queue = { SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, static_cast(kNumOfOpenSLESBuffers)}; SLDataSink audio_sink = {&buffer_queue, &pcm_format_}; // Create the audio recorder object (requires the RECORD_AUDIO permission). // Do not realize the recorder yet. Set the configuration first. const SLInterfaceID interface_id[] = {SL_IID_ANDROIDSIMPLEBUFFERQUEUE, SL_IID_ANDROIDCONFIGURATION}; const SLboolean interface_required[] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE}; if (LOG_ON_ERROR((*engine_)->CreateAudioRecorder( engine_, recorder_object_.Receive(), &audio_source, &audio_sink, arraysize(interface_id), interface_id, interface_required))) { return false; } // Configure the audio recorder (before it is realized). SLAndroidConfigurationItf recorder_config; if (LOG_ON_ERROR((recorder_object_->GetInterface(recorder_object_.Get(), SL_IID_ANDROIDCONFIGURATION, &recorder_config)))) { return false; } // Uses the default microphone tuned for audio communication. // Note that, SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION leads to a fast // track but also excludes usage of required effects like AEC, AGC and NS. // SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION SLint32 stream_type = SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION; if (LOG_ON_ERROR(((*recorder_config) ->SetConfiguration(recorder_config, SL_ANDROID_KEY_RECORDING_PRESET, &stream_type, sizeof(SLint32))))) { return false; } // The audio recorder can now be realized (in synchronous mode). if (LOG_ON_ERROR((recorder_object_->Realize(recorder_object_.Get(), SL_BOOLEAN_FALSE)))) { return false; } // Get the implicit recorder interface (SL_IID_RECORD). if (LOG_ON_ERROR((recorder_object_->GetInterface( recorder_object_.Get(), SL_IID_RECORD, &recorder_)))) { return false; } // Get the simple buffer queue interface (SL_IID_ANDROIDSIMPLEBUFFERQUEUE). // It was explicitly requested. if (LOG_ON_ERROR((recorder_object_->GetInterface( recorder_object_.Get(), SL_IID_ANDROIDSIMPLEBUFFERQUEUE, &simple_buffer_queue_)))) { return false; } // Register the input callback for the simple buffer queue. // This callback will be called when receiving new data from the device. if (LOG_ON_ERROR(((*simple_buffer_queue_) ->RegisterCallback(simple_buffer_queue_, SimpleBufferQueueCallback, this)))) { return false; } return true; } void OpenSLESRecorder::DestroyAudioRecorder() { ALOGD("DestroyAudioRecorder"); RTC_DCHECK(thread_checker_.IsCurrent()); if (!recorder_object_.Get()) return; (*simple_buffer_queue_) ->RegisterCallback(simple_buffer_queue_, nullptr, nullptr); recorder_object_.Reset(); recorder_ = nullptr; simple_buffer_queue_ = nullptr; } void OpenSLESRecorder::SimpleBufferQueueCallback( SLAndroidSimpleBufferQueueItf buffer_queue, void* context) { OpenSLESRecorder* stream = static_cast(context); stream->ReadBufferQueue(); } void OpenSLESRecorder::AllocateDataBuffers() { ALOGD("AllocateDataBuffers"); RTC_DCHECK(thread_checker_.IsCurrent()); RTC_DCHECK(!simple_buffer_queue_); RTC_CHECK(audio_device_buffer_); // Create a modified audio buffer class which allows us to deliver any number // of samples (and not only multiple of 10ms) to match the native audio unit // buffer size. ALOGD("frames per native buffer: %zu", audio_parameters_.frames_per_buffer()); ALOGD("frames per 10ms buffer: %zu", audio_parameters_.frames_per_10ms_buffer()); ALOGD("bytes per native buffer: %zu", audio_parameters_.GetBytesPerBuffer()); ALOGD("native sample rate: %d", audio_parameters_.sample_rate()); RTC_DCHECK(audio_device_buffer_); fine_audio_buffer_ = std::make_unique(audio_device_buffer_); // Allocate queue of audio buffers that stores recorded audio samples. const int buffer_size_samples = audio_parameters_.frames_per_buffer() * audio_parameters_.channels(); audio_buffers_.reset(new std::unique_ptr[kNumOfOpenSLESBuffers]); for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) { audio_buffers_[i].reset(new SLint16[buffer_size_samples]); } } void OpenSLESRecorder::ReadBufferQueue() { RTC_DCHECK(thread_checker_opensles_.IsCurrent()); SLuint32 state = GetRecordState(); if (state != SL_RECORDSTATE_RECORDING) { ALOGW("Buffer callback in non-recording state!"); return; } // Check delta time between two successive callbacks and provide a warning // if it becomes very large. // TODO(henrika): using 150ms as upper limit but this value is rather random. const uint32_t current_time = rtc::Time(); const uint32_t diff = current_time - last_rec_time_; if (diff > 150) { ALOGW("Bad OpenSL ES record timing, dT=%u [ms]", diff); } last_rec_time_ = current_time; // Send recorded audio data to the WebRTC sink. // TODO(henrika): fix delay estimates. It is OK to use fixed values for now // since there is no support to turn off built-in EC in combination with // OpenSL ES anyhow. Hence, as is, the WebRTC based AEC (which would use // these estimates) will never be active. fine_audio_buffer_->DeliverRecordedData( rtc::ArrayView( audio_buffers_[buffer_index_].get(), audio_parameters_.frames_per_buffer() * audio_parameters_.channels()), 25); // Enqueue the utilized audio buffer and use if for recording again. EnqueueAudioBuffer(); } bool OpenSLESRecorder::EnqueueAudioBuffer() { SLresult err = (*simple_buffer_queue_) ->Enqueue( simple_buffer_queue_, reinterpret_cast(audio_buffers_[buffer_index_].get()), audio_parameters_.GetBytesPerBuffer()); if (SL_RESULT_SUCCESS != err) { ALOGE("Enqueue failed: %s", GetSLErrorString(err)); return false; } buffer_index_ = (buffer_index_ + 1) % kNumOfOpenSLESBuffers; return true; } SLuint32 OpenSLESRecorder::GetRecordState() const { RTC_DCHECK(recorder_); SLuint32 state; SLresult err = (*recorder_)->GetRecordState(recorder_, &state); if (SL_RESULT_SUCCESS != err) { ALOGE("GetRecordState failed: %s", GetSLErrorString(err)); } return state; } SLAndroidSimpleBufferQueueState OpenSLESRecorder::GetBufferQueueState() const { RTC_DCHECK(simple_buffer_queue_); // state.count: Number of buffers currently in the queue. // state.index: Index of the currently filling buffer. This is a linear index // that keeps a cumulative count of the number of buffers recorded. SLAndroidSimpleBufferQueueState state; SLresult err = (*simple_buffer_queue_)->GetState(simple_buffer_queue_, &state); if (SL_RESULT_SUCCESS != err) { ALOGE("GetState failed: %s", GetSLErrorString(err)); } return state; } void OpenSLESRecorder::LogBufferState() const { SLAndroidSimpleBufferQueueState state = GetBufferQueueState(); ALOGD("state.count:%d state.index:%d", state.count, state.index); } SLuint32 OpenSLESRecorder::GetBufferCount() { SLAndroidSimpleBufferQueueState state = GetBufferQueueState(); return state.count; } } // namespace webrtc