summaryrefslogtreecommitdiffstats
path: root/dom/media/AudioStream.h
blob: 11a61b9fe71b877501991b6ef66624d634d2042e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#if !defined(AudioStream_h_)
#  define AudioStream_h_

#  include "AudioSampleFormat.h"
#  include "CubebUtils.h"
#  include "MediaInfo.h"
#  include "MediaSink.h"
#  include "mozilla/Atomics.h"
#  include "mozilla/Monitor.h"
#  include "mozilla/MozPromise.h"
#  include "mozilla/ProfilerUtils.h"
#  include "mozilla/RefPtr.h"
#  include "mozilla/Result.h"
#  include "mozilla/TimeStamp.h"
#  include "mozilla/UniquePtr.h"
#  include "mozilla/SPSCQueue.h"
#  include "nsCOMPtr.h"
#  include "nsThreadUtils.h"
#  include "WavDumper.h"

namespace mozilla {

struct CubebDestroyPolicy {
  void operator()(cubeb_stream* aStream) const {
    cubeb_stream_destroy(aStream);
  }
};

class AudioStream;
class FrameHistory;
class AudioConfig;
class RLBoxSoundTouch;

// A struct that contains the number of frames serviced or underrun by a
// callback, alongside the sample-rate for this callback (in case of playback
// rate change, it can be variable).
struct CallbackInfo {
  CallbackInfo() = default;
  CallbackInfo(uint32_t aServiced, uint32_t aUnderrun, uint32_t aOutputRate)
      : mServiced(aServiced), mUnderrun(aUnderrun), mOutputRate(aOutputRate) {}
  uint32_t mServiced = 0;
  uint32_t mUnderrun = 0;
  uint32_t mOutputRate = 0;
};

class AudioClock {
 public:
  explicit AudioClock(uint32_t aInRate);

  // Update the number of samples that has been written in the audio backend.
  // Called on the audio thread only.
  void UpdateFrameHistory(uint32_t aServiced, uint32_t aUnderrun,
                          bool aAudioThreadChanged);

  /**
   * @param aFrames The playback position in frames of the audio engine.
   * @return The playback position in frames of the stream,
   *         adjusted by playback rate changes and underrun frames.
   */
  int64_t GetPositionInFrames(int64_t aFrames);

  /**
   * @param frames The playback position in frames of the audio engine.
   * @return The playback position in microseconds of the stream,
   *         adjusted by playback rate changes and underrun frames.
   */
  int64_t GetPosition(int64_t frames);

  // Set the playback rate.
  // Called on the audio thread only.
  void SetPlaybackRate(double aPlaybackRate);
  // Get the current playback rate.
  // Called on the audio thread only.
  double GetPlaybackRate() const;
  // Set if we are preserving the pitch.
  // Called on the audio thread only.
  void SetPreservesPitch(bool aPreservesPitch);
  // Get the current pitch preservation state.
  // Called on the audio thread only.
  bool GetPreservesPitch() const;

  // Called on either thread.
  uint32_t GetInputRate() const { return mInRate; }
  uint32_t GetOutputRate() const { return mOutRate; }

 private:
  // Output rate in Hz (characteristic of the playback rate). Written on the
  // audio thread, read on either thread.
  Atomic<uint32_t> mOutRate;
  // Input rate in Hz (characteristic of the media being played).
  const uint32_t mInRate;
  // True if the we are timestretching, false if we are resampling. Accessed on
  // the audio thread only.
  bool mPreservesPitch;
  // The history of frames sent to the audio engine in each DataCallback.
  // Only accessed from non-audio threads on macOS, accessed on both threads and
  // protected by the AudioStream monitor on other platforms.
  const UniquePtr<FrameHistory> mFrameHistory
#  ifndef XP_MACOSX
      MOZ_GUARDED_BY(mMutex)
#  endif
          ;
#  ifdef XP_MACOSX
  // Enqueued on the audio thread, dequeued from the other thread. The maximum
  // size of this queue has been chosen empirically.
  SPSCQueue<CallbackInfo> mCallbackInfoQueue{100};
  // If it isn't possible to send the callback info to the non-audio thread,
  // store them here until it's possible to send them. This is an unlikely
  // fallback path. The size of this array has been chosen empirically. Only
  // ever accessed on the audio thread.
  AutoTArray<CallbackInfo, 5> mAudioThreadCallbackInfo;
#  else
  Mutex mMutex{"AudioClock"};
#  endif
};

/*
 * A bookkeeping class to track the read/write position of an audio buffer.
 */
class AudioBufferCursor {
 public:
  AudioBufferCursor(Span<AudioDataValue> aSpan, uint32_t aChannels,
                    uint32_t aFrames)
      : mChannels(aChannels), mSpan(aSpan), mFrames(aFrames) {}

  // Advance the cursor to account for frames that are consumed.
  uint32_t Advance(uint32_t aFrames) {
    MOZ_DIAGNOSTIC_ASSERT(Contains(aFrames));
    MOZ_ASSERT(mFrames >= aFrames);
    mFrames -= aFrames;
    mOffset += mChannels * aFrames;
    return aFrames;
  }

  // The number of frames available for read/write in this buffer.
  uint32_t Available() const { return mFrames; }

  // Return a pointer where read/write should begin.
  AudioDataValue* Ptr() const {
    MOZ_DIAGNOSTIC_ASSERT(mOffset <= mSpan.Length());
    return mSpan.Elements() + mOffset;
  }

 protected:
  bool Contains(uint32_t aFrames) const {
    return mSpan.Length() >= mOffset + mChannels * aFrames;
  }
  const uint32_t mChannels;

 private:
  const Span<AudioDataValue> mSpan;
  size_t mOffset = 0;
  uint32_t mFrames;
};

/*
 * A helper class to encapsulate pointer arithmetic and provide means to modify
 * the underlying audio buffer.
 */
class AudioBufferWriter : public AudioBufferCursor {
 public:
  AudioBufferWriter(Span<AudioDataValue> aSpan, uint32_t aChannels,
                    uint32_t aFrames)
      : AudioBufferCursor(aSpan, aChannels, aFrames) {}

  uint32_t WriteZeros(uint32_t aFrames) {
    MOZ_DIAGNOSTIC_ASSERT(Contains(aFrames));
    memset(Ptr(), 0, sizeof(AudioDataValue) * mChannels * aFrames);
    return Advance(aFrames);
  }

  uint32_t Write(const AudioDataValue* aPtr, uint32_t aFrames) {
    MOZ_DIAGNOSTIC_ASSERT(Contains(aFrames));
    memcpy(Ptr(), aPtr, sizeof(AudioDataValue) * mChannels * aFrames);
    return Advance(aFrames);
  }

  // Provide a write fuction to update the audio buffer with the following
  // signature: uint32_t(const AudioDataValue* aPtr, uint32_t aFrames)
  // aPtr: Pointer to the audio buffer.
  // aFrames: The number of frames available in the buffer.
  // return: The number of frames actually written by the function.
  template <typename Function>
  uint32_t Write(const Function& aFunction, uint32_t aFrames) {
    MOZ_DIAGNOSTIC_ASSERT(Contains(aFrames));
    return Advance(aFunction(Ptr(), aFrames));
  }

  using AudioBufferCursor::Available;
};

// Access to a single instance of this class must be synchronized by
// callers, or made from a single thread.  One exception is that access to
// GetPosition, GetPositionInFrames, SetVolume, and Get{Rate,Channels},
// SetMicrophoneActive is thread-safe without external synchronization.
class AudioStream final {
  virtual ~AudioStream();

 public:
  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AudioStream)

  class Chunk {
   public:
    // Return a pointer to the audio data.
    virtual const AudioDataValue* Data() const = 0;
    // Return the number of frames in this chunk.
    virtual uint32_t Frames() const = 0;
    // Return the number of audio channels.
    virtual uint32_t Channels() const = 0;
    // Return the sample rate of this chunk.
    virtual uint32_t Rate() const = 0;
    // Return a writable pointer for downmixing.
    virtual AudioDataValue* GetWritable() const = 0;
    virtual ~Chunk() = default;
  };

  class DataSource {
   public:
    // Attempt to acquire aFrames frames of audio, and returns the number of
    // frames successfuly acquired.
    virtual uint32_t PopFrames(AudioDataValue* aAudio, uint32_t aFrames,
                               bool aAudioThreadChanged) = 0;
    // Return true if no more data will be added to the source.
    virtual bool Ended() const = 0;

   protected:
    virtual ~DataSource() = default;
  };

  // aOutputChannels is the number of audio channels (1 for mono, 2 for stereo,
  // etc), aChannelMap is the indicator for channel layout(mono, stereo, 5.1 or
  // 7.1 ). Initialize the audio stream.and aRate is the sample rate
  // (22050Hz, 44100Hz, etc).
  AudioStream(DataSource& aSource, uint32_t aInRate, uint32_t aOutputChannels,
              AudioConfig::ChannelLayout::ChannelMap aChannelMap);

  nsresult Init(AudioDeviceInfo* aSinkInfo);

  // Closes the stream. All future use of the stream is an error.
  void ShutDown();

  // Set the current volume of the audio playback. This is a value from
  // 0 (meaning muted) to 1 (meaning full volume).  Thread-safe.
  void SetVolume(double aVolume);

  void SetStreamName(const nsAString& aStreamName);

  // Start the stream.
  RefPtr<MediaSink::EndedPromise> Start();

  // Pause audio playback.
  void Pause();

  // Resume audio playback.
  void Resume();

  // Return the position in microseconds of the audio frame being played by
  // the audio hardware, compensated for playback rate change. Thread-safe.
  int64_t GetPosition();

  // Return the position, measured in audio frames played since the stream
  // was opened, of the audio hardware.  Thread-safe.
  int64_t GetPositionInFrames();

  uint32_t GetOutChannels() const { return mOutChannels; }

  // Set playback rate as a multiple of the intrinsic playback rate. This is
  // to be called only with aPlaybackRate > 0.0.
  nsresult SetPlaybackRate(double aPlaybackRate);
  // Switch between resampling (if false) and time stretching (if true,
  // default).
  nsresult SetPreservesPitch(bool aPreservesPitch);

  size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const;

  bool IsPlaybackCompleted() const;

  // Returns true if at least one DataCallback has been called.
  bool CallbackStarted() const { return mCallbacksStarted; }

 protected:
  friend class AudioClock;

  // Return the position, measured in audio frames played since the stream was
  // opened, of the audio hardware, not adjusted for the changes of playback
  // rate or underrun frames.
  // Caller must own the monitor.
  int64_t GetPositionInFramesUnlocked();

 private:
  nsresult OpenCubeb(cubeb* aContext, cubeb_stream_params& aParams,
                     TimeStamp aStartTime, bool aIsFirst);

  static long DataCallback_S(cubeb_stream*, void* aThis,
                             const void* /* aInputBuffer */,
                             void* aOutputBuffer, long aFrames) {
    return static_cast<AudioStream*>(aThis)->DataCallback(aOutputBuffer,
                                                          aFrames);
  }

  static void StateCallback_S(cubeb_stream*, void* aThis, cubeb_state aState) {
    static_cast<AudioStream*>(aThis)->StateCallback(aState);
  }

  long DataCallback(void* aBuffer, long aFrames);
  void StateCallback(cubeb_state aState);

  // Audio thread only
  nsresult EnsureTimeStretcherInitialized();
  void GetUnprocessed(AudioBufferWriter& aWriter);
  void GetTimeStretched(AudioBufferWriter& aWriter);
  void UpdatePlaybackRateIfNeeded();

  // Return true if audio frames are valid (correct sampling rate and valid
  // channel count) otherwise false.
  bool IsValidAudioFormat(Chunk* aChunk) MOZ_REQUIRES(mMonitor);

  template <typename Function, typename... Args>
  int InvokeCubeb(Function aFunction, Args&&... aArgs) MOZ_REQUIRES(mMonitor);
  bool CheckThreadIdChanged();
  void AssertIsOnAudioThread() const;

  RLBoxSoundTouch* mTimeStretcher;
  AudioClock mAudioClock;

  WavDumper mDumpFile;

  const AudioConfig::ChannelLayout::ChannelMap mChannelMap;

  // The monitor is held to protect all access to member variables below.
  Monitor mMonitor MOZ_UNANNOTATED;

  const uint32_t mOutChannels;

  // mCubebStream holds a bare pointer to cubeb, so we hold a ref on its behalf
  RefPtr<CubebUtils::CubebHandle> mCubeb;
  // Owning reference to a cubeb_stream.  Set in Init(), cleared in ShutDown, so
  // no lock is needed to access.
  UniquePtr<cubeb_stream, CubebDestroyPolicy> mCubebStream;

  enum StreamState {
    INITIALIZED,  // Initialized, playback has not begun.
    STARTED,      // cubeb started.
    STOPPED,      // Stopped by a call to Pause().
    DRAINED,      // StateCallback has indicated that the drain is complete.
    ERRORED,      // Stream disabled due to an internal error.
    SHUTDOWN      // ShutDown has been called
  };

  std::atomic<StreamState> mState;

  // DataSource::PopFrames can never be called concurrently.
  // DataSource::IsEnded uses only atomics.
  DataSource& mDataSource;

  // The device info of the current sink. If null
  // the default device is used. It is set
  // during the Init() in decoder thread.
  RefPtr<AudioDeviceInfo> mSinkInfo;
  // Contains the id of the audio thread, from profiler_get_thread_id.
  std::atomic<ProfilerThreadId> mAudioThreadId;
  const bool mSandboxed = false;

  MozPromiseHolder<MediaSink::EndedPromise> mEndedPromise
      MOZ_GUARDED_BY(mMonitor);
  std::atomic<bool> mPlaybackComplete;
  // Both written on the MDSM thread, read on the audio thread.
  std::atomic<float> mPlaybackRate;
  std::atomic<bool> mPreservesPitch;
  // Audio thread only
  bool mAudioThreadChanged = false;
  Atomic<bool> mCallbacksStarted;
};

}  // namespace mozilla

#endif