1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
|
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "GraphDriver.h"
#include "AudioNodeEngine.h"
#include "cubeb/cubeb.h"
#include "mozilla/dom/AudioContext.h"
#include "mozilla/dom/AudioDeviceInfo.h"
#include "mozilla/dom/BaseAudioContextBinding.h"
#include "mozilla/SchedulerGroup.h"
#include "mozilla/SharedThreadPool.h"
#include "mozilla/ClearOnShutdown.h"
#include "mozilla/Unused.h"
#include "mozilla/MathAlgorithms.h"
#include "mozilla/StaticPrefs_media.h"
#include "CubebDeviceEnumerator.h"
#include "MediaTrackGraphImpl.h"
#include "CallbackThreadRegistry.h"
#include "Tracing.h"
#ifdef MOZ_WEBRTC
# include "webrtc/MediaEngineWebRTC.h"
#endif
#ifdef XP_MACOSX
# include <sys/sysctl.h>
# include "nsCocoaFeatures.h"
#endif
extern mozilla::LazyLogModule gMediaTrackGraphLog;
#ifdef LOG
# undef LOG
#endif // LOG
#define LOG(type, msg) MOZ_LOG(gMediaTrackGraphLog, type, msg)
namespace mozilla {
GraphDriver::GraphDriver(GraphInterface* aGraphInterface,
GraphDriver* aPreviousDriver, uint32_t aSampleRate)
: mGraphInterface(aGraphInterface),
mSampleRate(aSampleRate),
mPreviousDriver(aPreviousDriver) {}
void GraphDriver::SetStreamName(const nsACString& aStreamName) {
MOZ_ASSERT(InIteration() || (!ThreadRunning() && NS_IsMainThread()));
mStreamName = aStreamName;
LOG(LogLevel::Debug, ("%p: GraphDriver::SetStreamName driver=%p %s", Graph(),
this, mStreamName.get()));
}
void GraphDriver::SetState(const nsACString& aStreamName,
GraphTime aIterationEnd,
GraphTime aStateComputedTime) {
MOZ_ASSERT(InIteration() || !ThreadRunning());
mStreamName = aStreamName;
mIterationEnd = aIterationEnd;
mStateComputedTime = aStateComputedTime;
}
#ifdef DEBUG
bool GraphDriver::InIteration() const {
return OnThread() || Graph()->InDriverIteration(this);
}
#endif
GraphDriver* GraphDriver::PreviousDriver() {
MOZ_ASSERT(InIteration() || !ThreadRunning());
return mPreviousDriver;
}
void GraphDriver::SetPreviousDriver(GraphDriver* aPreviousDriver) {
MOZ_ASSERT(InIteration() || !ThreadRunning());
mPreviousDriver = aPreviousDriver;
}
ThreadedDriver::ThreadedDriver(GraphInterface* aGraphInterface,
GraphDriver* aPreviousDriver,
uint32_t aSampleRate)
: GraphDriver(aGraphInterface, aPreviousDriver, aSampleRate),
mThreadRunning(false) {}
class MediaTrackGraphShutdownThreadRunnable : public Runnable {
public:
explicit MediaTrackGraphShutdownThreadRunnable(
already_AddRefed<nsIThread> aThread)
: Runnable("MediaTrackGraphShutdownThreadRunnable"), mThread(aThread) {}
NS_IMETHOD Run() override {
TRACE("MediaTrackGraphShutdownThreadRunnable");
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(mThread);
mThread->AsyncShutdown();
mThread = nullptr;
return NS_OK;
}
private:
nsCOMPtr<nsIThread> mThread;
};
ThreadedDriver::~ThreadedDriver() {
if (mThread) {
nsCOMPtr<nsIRunnable> event =
new MediaTrackGraphShutdownThreadRunnable(mThread.forget());
SchedulerGroup::Dispatch(event.forget());
}
}
class MediaTrackGraphInitThreadRunnable : public Runnable {
public:
explicit MediaTrackGraphInitThreadRunnable(ThreadedDriver* aDriver)
: Runnable("MediaTrackGraphInitThreadRunnable"), mDriver(aDriver) {}
NS_IMETHOD Run() override {
TRACE("MediaTrackGraphInitThreadRunnable");
MOZ_ASSERT(!mDriver->ThreadRunning());
LOG(LogLevel::Debug, ("Starting a new system driver for graph %p",
mDriver->mGraphInterface.get()));
if (GraphDriver* previousDriver = mDriver->PreviousDriver()) {
LOG(LogLevel::Debug,
("%p releasing an AudioCallbackDriver(%p), for graph %p",
mDriver.get(), previousDriver, mDriver->Graph()));
MOZ_ASSERT(!mDriver->AsAudioCallbackDriver());
AudioCallbackDriver* audioCallbackDriver =
previousDriver->AsAudioCallbackDriver();
MOZ_ALWAYS_SUCCEEDS(audioCallbackDriver->mCubebOperationThread->Dispatch(
NS_NewRunnableFunction(
"ThreadedDriver previousDriver::Stop()",
[audioCallbackDriver = RefPtr{audioCallbackDriver}] {
audioCallbackDriver->Stop();
})));
mDriver->SetPreviousDriver(nullptr);
}
mDriver->RunThread();
return NS_OK;
}
private:
RefPtr<ThreadedDriver> mDriver;
};
void ThreadedDriver::Start() {
MOZ_ASSERT(!ThreadRunning());
LOG(LogLevel::Debug,
("Starting thread for a SystemClockDriver %p", mGraphInterface.get()));
Unused << NS_WARN_IF(mThread);
MOZ_ASSERT(!mThread); // Ensure we haven't already started it
nsCOMPtr<nsIRunnable> event = new MediaTrackGraphInitThreadRunnable(this);
// Note: mThread may be null during event->Run() if we pass to NewNamedThread!
// See AudioInitTask
nsresult rv = NS_NewNamedThread("MediaTrackGrph", getter_AddRefs(mThread));
if (NS_SUCCEEDED(rv)) {
mThread->Dispatch(event.forget(), NS_DISPATCH_NORMAL);
}
}
void ThreadedDriver::Shutdown() {
NS_ASSERTION(NS_IsMainThread(), "Must be called on main thread");
// mGraph's thread is not running so it's OK to do whatever here
LOG(LogLevel::Debug, ("Stopping threads for MediaTrackGraph %p", this));
if (mThread) {
LOG(LogLevel::Debug,
("%p: Stopping ThreadedDriver's %p thread", Graph(), this));
mThread->AsyncShutdown();
mThread = nullptr;
}
}
SystemClockDriver::SystemClockDriver(GraphInterface* aGraphInterface,
GraphDriver* aPreviousDriver,
uint32_t aSampleRate)
: ThreadedDriver(aGraphInterface, aPreviousDriver, aSampleRate),
mInitialTimeStamp(TimeStamp::Now()),
mCurrentTimeStamp(TimeStamp::Now()),
mLastTimeStamp(TimeStamp::Now()) {}
SystemClockDriver::~SystemClockDriver() = default;
void ThreadedDriver::RunThread() {
mThreadRunning = true;
while (true) {
auto iterationStart = mIterationEnd;
mIterationEnd += GetIntervalForIteration();
if (mStateComputedTime < mIterationEnd) {
LOG(LogLevel::Warning, ("%p: Global underrun detected", Graph()));
mIterationEnd = mStateComputedTime;
}
if (iterationStart >= mIterationEnd) {
NS_ASSERTION(iterationStart == mIterationEnd, "Time can't go backwards!");
// This could happen due to low clock resolution, maybe?
LOG(LogLevel::Debug, ("%p: Time did not advance", Graph()));
}
GraphTime nextStateComputedTime =
MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(
mIterationEnd + MillisecondsToMediaTime(AUDIO_TARGET_MS));
if (nextStateComputedTime < mStateComputedTime) {
// A previous driver may have been processing further ahead of
// iterationEnd.
LOG(LogLevel::Warning,
("%p: Prevent state from going backwards. interval[%ld; %ld] "
"state[%ld; "
"%ld]",
Graph(), (long)iterationStart, (long)mIterationEnd,
(long)mStateComputedTime, (long)nextStateComputedTime));
nextStateComputedTime = mStateComputedTime;
}
LOG(LogLevel::Verbose,
("%p: interval[%ld; %ld] state[%ld; %ld]", Graph(),
(long)iterationStart, (long)mIterationEnd, (long)mStateComputedTime,
(long)nextStateComputedTime));
mStateComputedTime = nextStateComputedTime;
IterationResult result =
Graph()->OneIteration(mStateComputedTime, mIterationEnd, nullptr);
if (result.IsStop()) {
// Signal that we're done stopping.
result.Stopped();
break;
}
WaitForNextIteration();
if (GraphDriver* nextDriver = result.NextDriver()) {
LOG(LogLevel::Debug, ("%p: Switching to AudioCallbackDriver", Graph()));
result.Switched();
nextDriver->SetState(mStreamName, mIterationEnd, mStateComputedTime);
nextDriver->Start();
break;
}
MOZ_ASSERT(result.IsStillProcessing());
}
mThreadRunning = false;
}
MediaTime SystemClockDriver::GetIntervalForIteration() {
TimeStamp now = TimeStamp::Now();
MediaTime interval =
SecondsToMediaTime((now - mCurrentTimeStamp).ToSeconds());
mCurrentTimeStamp = now;
MOZ_LOG(gMediaTrackGraphLog, LogLevel::Verbose,
("%p: Updating current time to %f (real %f, StateComputedTime() %f)",
Graph(), MediaTimeToSeconds(mIterationEnd + interval),
(now - mInitialTimeStamp).ToSeconds(),
MediaTimeToSeconds(mStateComputedTime)));
return interval;
}
void ThreadedDriver::EnsureNextIteration() {
mWaitHelper.EnsureNextIteration();
}
void ThreadedDriver::WaitForNextIteration() {
MOZ_ASSERT(mThread);
MOZ_ASSERT(OnThread());
mWaitHelper.WaitForNextIterationAtLeast(WaitInterval());
}
TimeDuration SystemClockDriver::WaitInterval() {
MOZ_ASSERT(mThread);
MOZ_ASSERT(OnThread());
TimeStamp now = TimeStamp::Now();
int64_t timeoutMS = MEDIA_GRAPH_TARGET_PERIOD_MS -
int64_t((now - mCurrentTimeStamp).ToMilliseconds());
// Make sure timeoutMS doesn't overflow 32 bits by waking up at
// least once a minute, if we need to wake up at all
timeoutMS = std::max<int64_t>(0, std::min<int64_t>(timeoutMS, 60 * 1000));
LOG(LogLevel::Verbose,
("%p: Waiting for next iteration; at %f, timeout=%f", Graph(),
(now - mInitialTimeStamp).ToSeconds(), timeoutMS / 1000.0));
return TimeDuration::FromMilliseconds(timeoutMS);
}
OfflineClockDriver::OfflineClockDriver(GraphInterface* aGraphInterface,
uint32_t aSampleRate, GraphTime aSlice)
: ThreadedDriver(aGraphInterface, nullptr, aSampleRate), mSlice(aSlice) {}
OfflineClockDriver::~OfflineClockDriver() = default;
void OfflineClockDriver::RunThread() {
nsCOMPtr<nsIThreadInternal> threadInternal = do_QueryInterface(mThread);
nsCOMPtr<nsIThreadObserver> observer = do_QueryInterface(Graph());
threadInternal->SetObserver(observer);
ThreadedDriver::RunThread();
}
MediaTime OfflineClockDriver::GetIntervalForIteration() {
return MillisecondsToMediaTime(mSlice);
}
/* Helper to proxy the GraphInterface methods used by a running
* mFallbackDriver. */
class AudioCallbackDriver::FallbackWrapper : public GraphInterface {
public:
FallbackWrapper(RefPtr<GraphInterface> aGraph,
RefPtr<AudioCallbackDriver> aOwner, uint32_t aSampleRate,
const nsACString& aStreamName, GraphTime aIterationEnd,
GraphTime aStateComputedTime)
: mGraph(std::move(aGraph)),
mOwner(std::move(aOwner)),
mFallbackDriver(
MakeRefPtr<SystemClockDriver>(this, nullptr, aSampleRate)) {
mFallbackDriver->SetState(aStreamName, aIterationEnd, aStateComputedTime);
}
NS_DECL_THREADSAFE_ISUPPORTS
/* Proxied SystemClockDriver methods */
void Start() { mFallbackDriver->Start(); }
MOZ_CAN_RUN_SCRIPT void Shutdown() {
RefPtr<SystemClockDriver> driver = mFallbackDriver;
driver->Shutdown();
}
void SetStreamName(const nsACString& aStreamName) {
mFallbackDriver->SetStreamName(aStreamName);
}
void EnsureNextIteration() { mFallbackDriver->EnsureNextIteration(); }
#ifdef DEBUG
bool InIteration() { return mFallbackDriver->InIteration(); }
#endif
bool OnThread() { return mFallbackDriver->OnThread(); }
/* GraphInterface methods */
void NotifyInputStopped() override {
MOZ_CRASH("Unexpected NotifyInputStopped from fallback SystemClockDriver");
}
void NotifyInputData(const AudioDataValue* aBuffer, size_t aFrames,
TrackRate aRate, uint32_t aChannels,
uint32_t aAlreadyBuffered) override {
MOZ_CRASH("Unexpected NotifyInputData from fallback SystemClockDriver");
}
void NotifySetRequestedInputProcessingParamsResult(
AudioCallbackDriver* aDriver,
cubeb_input_processing_params aRequestedParams,
Result<cubeb_input_processing_params, int>&& aResult) override {
MOZ_CRASH(
"Unexpected processing params result from fallback SystemClockDriver");
}
void DeviceChanged() override {
MOZ_CRASH("Unexpected DeviceChanged from fallback SystemClockDriver");
}
#ifdef DEBUG
bool InDriverIteration(const GraphDriver* aDriver) const override {
return mGraph->InDriverIteration(mOwner) && mOwner->OnFallback();
}
#endif
IterationResult OneIteration(GraphTime aStateComputedEnd,
GraphTime aIterationEnd,
MixerCallbackReceiver* aMixerReceiver) override {
MOZ_ASSERT(!aMixerReceiver);
#ifdef DEBUG
AutoInCallback aic(mOwner);
#endif
IterationResult result =
mGraph->OneIteration(aStateComputedEnd, aIterationEnd, aMixerReceiver);
AudioStreamState audioState = mOwner->mAudioStreamState;
MOZ_ASSERT(audioState != AudioStreamState::Stopping,
"The audio driver can only enter stopping if it iterated the "
"graph, which it can only do if there's no fallback driver");
// After a devicechange event from the audio driver, wait for a five
// millisecond grace period before handing control to the audio driver. We
// do this because cubeb leaves no guarantee on audio callbacks coming in
// after a device change event.
if (audioState == AudioStreamState::ChangingDevice &&
mOwner->mChangingDeviceStartTime + TimeDuration::FromMilliseconds(5) <
TimeStamp::Now()) {
mOwner->mChangingDeviceStartTime = TimeStamp();
if (mOwner->mAudioStreamState.compareExchange(
AudioStreamState::ChangingDevice, AudioStreamState::Starting)) {
audioState = AudioStreamState::Starting;
LOG(LogLevel::Debug, ("%p: Fallback driver has started. Waiting for "
"audio driver to start.",
mOwner.get()));
}
}
if (audioState != AudioStreamState::Running && result.IsStillProcessing()) {
mOwner->MaybeStartAudioStream();
return result;
}
MOZ_ASSERT(result.IsStillProcessing() || result.IsStop() ||
result.IsSwitchDriver());
// Proxy the release of the fallback driver to a background thread, so it
// doesn't perform unexpected suicide.
IterationResult stopFallback =
IterationResult::CreateStop(NS_NewRunnableFunction(
"AudioCallbackDriver::FallbackDriverStopped",
[self = RefPtr<FallbackWrapper>(this), this, aIterationEnd,
aStateComputedEnd, result = std::move(result)]() mutable {
FallbackDriverState fallbackState =
result.IsStillProcessing() ? FallbackDriverState::None
: FallbackDriverState::Stopped;
mOwner->FallbackDriverStopped(aIterationEnd, aStateComputedEnd,
fallbackState);
if (fallbackState == FallbackDriverState::Stopped) {
#ifdef DEBUG
// The AudioCallbackDriver may not iterate the graph, but we'll
// call into it so we need to be regarded as "in iteration".
AutoInCallback aic(mOwner);
#endif
if (GraphDriver* nextDriver = result.NextDriver()) {
LOG(LogLevel::Debug,
("%p: Switching from fallback to other driver.",
mOwner.get()));
result.Switched();
nextDriver->SetState(mOwner->mStreamName, aIterationEnd,
aStateComputedEnd);
nextDriver->Start();
} else if (result.IsStop()) {
LOG(LogLevel::Debug,
("%p: Stopping fallback driver.", mOwner.get()));
result.Stopped();
}
}
mOwner = nullptr;
NS_DispatchBackgroundTask(NS_NewRunnableFunction(
"AudioCallbackDriver::FallbackDriverStopped::Release",
[fallback = std::move(self->mFallbackDriver)] {}));
}));
return stopFallback;
}
private:
virtual ~FallbackWrapper() = default;
const RefPtr<GraphInterface> mGraph;
// Valid until mFallbackDriver has finished its last iteration.
RefPtr<AudioCallbackDriver> mOwner;
RefPtr<SystemClockDriver> mFallbackDriver;
};
NS_IMPL_ISUPPORTS0(AudioCallbackDriver::FallbackWrapper)
/* static */
already_AddRefed<TaskQueue> AudioCallbackDriver::CreateTaskQueue() {
return TaskQueue::Create(CubebUtils::GetCubebOperationThread(),
"AudioCallbackDriver cubeb task queue")
.forget();
}
AudioCallbackDriver::AudioCallbackDriver(
GraphInterface* aGraphInterface, GraphDriver* aPreviousDriver,
uint32_t aSampleRate, uint32_t aOutputChannelCount,
uint32_t aInputChannelCount, CubebUtils::AudioDeviceID aOutputDeviceID,
CubebUtils::AudioDeviceID aInputDeviceID, AudioInputType aAudioInputType,
cubeb_input_processing_params aRequestedInputProcessingParams)
: GraphDriver(aGraphInterface, aPreviousDriver, aSampleRate),
mOutputChannelCount(aOutputChannelCount),
mInputChannelCount(aInputChannelCount),
mOutputDeviceID(aOutputDeviceID),
mInputDeviceID(aInputDeviceID),
mIterationDurationMS(MEDIA_GRAPH_TARGET_PERIOD_MS),
mCubebOperationThread(CreateTaskQueue()),
mRequestedInputProcessingParams(aRequestedInputProcessingParams),
mAudioThreadId(ProfilerThreadId{}),
mAudioThreadIdInCb(std::thread::id()),
mFallback("AudioCallbackDriver::mFallback"),
mSandboxed(CubebUtils::SandboxEnabled()) {
LOG(LogLevel::Debug, ("%p: AudioCallbackDriver %p ctor - input: device %p, "
"channel %d, output: device %p, channel %d",
Graph(), this, mInputDeviceID, mInputChannelCount,
mOutputDeviceID, mOutputChannelCount));
NS_WARNING_ASSERTION(mOutputChannelCount != 0,
"Invalid output channel count");
MOZ_ASSERT(mOutputChannelCount <= 8);
if (aAudioInputType == AudioInputType::Voice &&
StaticPrefs::
media_getusermedia_microphone_prefer_voice_stream_with_processing_enabled()) {
LOG(LogLevel::Debug,
("%p: AudioCallbackDriver %p ctor - using VOICE and requesting input "
"processing params %s.",
Graph(), this,
CubebUtils::ProcessingParamsToString(aRequestedInputProcessingParams)
.get()));
mInputDevicePreference = CUBEB_DEVICE_PREF_VOICE;
CubebUtils::SetInCommunication(true);
} else {
mInputDevicePreference = CUBEB_DEVICE_PREF_ALL;
}
}
AudioCallbackDriver::~AudioCallbackDriver() {
if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
CubebUtils::SetInCommunication(false);
}
}
bool IsMacbookOrMacbookAir() {
#ifdef XP_MACOSX
size_t len = 0;
sysctlbyname("hw.model", NULL, &len, NULL, 0);
if (len) {
UniquePtr<char[]> model(new char[len]);
// This string can be
// MacBook%d,%d for a normal MacBook
// MacBookAir%d,%d for a Macbook Air
sysctlbyname("hw.model", model.get(), &len, NULL, 0);
char* substring = strstr(model.get(), "MacBook");
if (substring) {
const size_t offset = strlen("MacBook");
if (!strncmp(model.get() + offset, "Air", 3) ||
isdigit(model[offset + 1])) {
return true;
}
}
}
#endif
return false;
}
void AudioCallbackDriver::Init(const nsCString& aStreamName) {
LOG(LogLevel::Debug,
("%p: AudioCallbackDriver::Init driver=%p", Graph(), this));
TRACE("AudioCallbackDriver::Init");
MOZ_ASSERT(OnCubebOperationThread());
MOZ_ASSERT(mAudioStreamState == AudioStreamState::Pending);
if (mFallbackDriverState == FallbackDriverState::Stopped) {
// The graph has already stopped us.
return;
}
RefPtr<CubebUtils::CubebHandle> handle = CubebUtils::GetCubeb();
if (!handle) {
NS_WARNING("Could not get cubeb context.");
LOG(LogLevel::Warning, ("%s: Could not get cubeb context", __func__));
mAudioStreamState = AudioStreamState::None;
if (TryStartingFallbackDriver().isOk()) {
CubebUtils::ReportCubebStreamInitFailure(true);
}
return;
}
cubeb_stream_params output;
cubeb_stream_params input;
bool firstStream = CubebUtils::GetFirstStream();
MOZ_ASSERT(!NS_IsMainThread(),
"This is blocking and should never run on the main thread.");
output.rate = mSampleRate;
output.format = CUBEB_SAMPLE_FLOAT32NE;
if (!mOutputChannelCount) {
LOG(LogLevel::Warning, ("Output number of channels is 0."));
mAudioStreamState = AudioStreamState::None;
if (TryStartingFallbackDriver().isOk()) {
CubebUtils::ReportCubebStreamInitFailure(firstStream);
}
return;
}
CubebUtils::AudioDeviceID forcedOutputDeviceId = nullptr;
char* forcedOutputDeviceName = CubebUtils::GetForcedOutputDevice();
if (forcedOutputDeviceName) {
RefPtr<CubebDeviceEnumerator> enumerator = Enumerator::GetInstance();
RefPtr<AudioDeviceInfo> device = enumerator->DeviceInfoFromName(
NS_ConvertUTF8toUTF16(forcedOutputDeviceName), EnumeratorSide::OUTPUT);
if (device && device->DeviceID()) {
forcedOutputDeviceId = device->DeviceID();
}
}
mBuffer = AudioCallbackBufferWrapper<AudioDataValue>(mOutputChannelCount);
mScratchBuffer =
SpillBuffer<AudioDataValue, WEBAUDIO_BLOCK_SIZE * 2>(mOutputChannelCount);
output.channels = mOutputChannelCount;
AudioConfig::ChannelLayout::ChannelMap channelMap =
AudioConfig::ChannelLayout(mOutputChannelCount).Map();
output.layout = static_cast<uint32_t>(channelMap);
output.prefs = CubebUtils::GetDefaultStreamPrefs(CUBEB_DEVICE_TYPE_OUTPUT);
if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE &&
CubebUtils::RouteOutputAsVoice()) {
output.prefs |= static_cast<cubeb_stream_prefs>(CUBEB_STREAM_PREF_VOICE);
}
uint32_t latencyFrames = CubebUtils::GetCubebMTGLatencyInFrames(&output);
LOG(LogLevel::Debug, ("Minimum latency in frames: %d", latencyFrames));
// Macbook and MacBook air don't have enough CPU to run very low latency
// MediaTrackGraphs, cap the minimal latency to 512 frames int this case.
if (IsMacbookOrMacbookAir()) {
latencyFrames = std::max((uint32_t)512, latencyFrames);
LOG(LogLevel::Debug,
("Macbook or macbook air, new latency: %d", latencyFrames));
}
// Buffer sizes lower than 10ms are nowadays common. It's not very useful
// when doing voice, because all the WebRTC code that does audio input
// processing deals in 10ms chunks of audio. Take the first power of two
// above 10ms at the current rate in this case. It's probably 512, for common
// rates.
if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
if (latencyFrames < mSampleRate / 100) {
latencyFrames = mozilla::RoundUpPow2(mSampleRate / 100);
LOG(LogLevel::Debug,
("AudioProcessing enabled, new latency %d", latencyFrames));
}
}
// It's not useful for the graph to run with a block size lower than the Web
// Audio API block size, but increasingly devices report that they can do
// audio latencies lower than that.
if (latencyFrames < WEBAUDIO_BLOCK_SIZE) {
LOG(LogLevel::Debug,
("Latency clamped to %d from %d", WEBAUDIO_BLOCK_SIZE, latencyFrames));
latencyFrames = WEBAUDIO_BLOCK_SIZE;
}
LOG(LogLevel::Debug, ("Effective latency in frames: %d", latencyFrames));
input = output;
input.channels = mInputChannelCount;
input.layout = CUBEB_LAYOUT_UNDEFINED;
input.prefs = CubebUtils::GetDefaultStreamPrefs(CUBEB_DEVICE_TYPE_INPUT);
if (mInputDevicePreference == CUBEB_DEVICE_PREF_VOICE) {
input.prefs |= static_cast<cubeb_stream_prefs>(CUBEB_STREAM_PREF_VOICE);
}
cubeb_stream* stream = nullptr;
const char* streamName =
aStreamName.IsEmpty() ? "AudioCallbackDriver" : aStreamName.get();
bool inputWanted = mInputChannelCount > 0;
CubebUtils::AudioDeviceID outputId = mOutputDeviceID;
CubebUtils::AudioDeviceID inputId = mInputDeviceID;
if (CubebUtils::CubebStreamInit(
handle->Context(), &stream, streamName, inputId,
inputWanted ? &input : nullptr,
forcedOutputDeviceId ? forcedOutputDeviceId : outputId, &output,
latencyFrames, DataCallback_s, StateCallback_s, this) == CUBEB_OK) {
mCubeb = handle;
mAudioStream.own(stream);
DebugOnly<int> rv =
cubeb_stream_set_volume(mAudioStream, CubebUtils::GetVolumeScale());
NS_WARNING_ASSERTION(
rv == CUBEB_OK,
"Could not set the audio stream volume in GraphDriver.cpp");
CubebUtils::ReportCubebBackendUsed();
} else {
NS_WARNING(
"Could not create a cubeb stream for MediaTrackGraph, falling "
"back to a SystemClockDriver");
mAudioStreamState = AudioStreamState::None;
// Only report failures when we're not coming from a driver that was
// created itself as a fallback driver because of a previous audio driver
// failure.
if (TryStartingFallbackDriver().isOk()) {
CubebUtils::ReportCubebStreamInitFailure(firstStream);
}
return;
}
#ifdef XP_MACOSX
PanOutputIfNeeded(inputWanted);
#endif
if (inputWanted && InputDevicePreference() == AudioInputType::Voice) {
SetInputProcessingParams(mRequestedInputProcessingParams);
}
cubeb_stream_register_device_changed_callback(
mAudioStream, AudioCallbackDriver::DeviceChangedCallback_s);
// No-op if MOZ_DUMP_AUDIO is not defined as an environment variable. This
// is intended for diagnosing issues, and only works if the content sandbox is
// disabled.
mInputStreamFile.Open("GraphDriverInput", input.channels, input.rate);
mOutputStreamFile.Open("GraphDriverOutput", output.channels, output.rate);
if (NS_WARN_IF(!StartStream())) {
LOG(LogLevel::Warning,
("%p: AudioCallbackDriver couldn't start a cubeb stream.", Graph()));
return;
}
LOG(LogLevel::Debug, ("%p: AudioCallbackDriver started.", Graph()));
}
void AudioCallbackDriver::SetCubebStreamName(const nsCString& aStreamName) {
MOZ_ASSERT(OnCubebOperationThread());
MOZ_ASSERT(mAudioStream);
cubeb_stream_set_name(mAudioStream, aStreamName.get());
}
void AudioCallbackDriver::Start() {
MOZ_ASSERT(!IsStarted());
MOZ_ASSERT(mAudioStreamState == AudioStreamState::None);
MOZ_ASSERT_IF(PreviousDriver(), PreviousDriver()->InIteration());
mAudioStreamState = AudioStreamState::Pending;
// Starting an audio driver could take a while. We start a system driver in
// the meantime so that the graph is kept running.
(void)TryStartingFallbackDriver();
if (mPreviousDriver) {
if (AudioCallbackDriver* previousAudioCallback =
mPreviousDriver->AsAudioCallbackDriver()) {
LOG(LogLevel::Debug, ("Releasing audio driver off main thread."));
MOZ_ALWAYS_SUCCEEDS(
previousAudioCallback->mCubebOperationThread->Dispatch(
NS_NewRunnableFunction(
"AudioCallbackDriver previousDriver::Stop()",
[previousDriver = RefPtr{previousAudioCallback}] {
previousDriver->Stop();
})));
} else {
LOG(LogLevel::Debug,
("Dropping driver reference for SystemClockDriver."));
MOZ_ASSERT(mPreviousDriver->AsSystemClockDriver());
}
mPreviousDriver = nullptr;
}
LOG(LogLevel::Debug, ("Starting new audio driver off main thread, "
"to ensure it runs after previous shutdown."));
MOZ_ALWAYS_SUCCEEDS(mCubebOperationThread->Dispatch(
NS_NewRunnableFunction("AudioCallbackDriver Init()",
[self = RefPtr{this}, streamName = mStreamName] {
self->Init(streamName);
})));
}
bool AudioCallbackDriver::StartStream() {
TRACE("AudioCallbackDriver::StartStream");
MOZ_ASSERT(!IsStarted() && OnCubebOperationThread());
// Set STARTING before cubeb_stream_start, since starting the cubeb stream
// can result in a callback (that may read mAudioStreamState) before
// mAudioStreamState would otherwise be set.
mAudioStreamState = AudioStreamState::Starting;
if (cubeb_stream_start(mAudioStream) != CUBEB_OK) {
NS_WARNING("Could not start cubeb stream for MTG.");
return false;
}
return true;
}
void AudioCallbackDriver::Stop() {
LOG(LogLevel::Debug,
("%p: AudioCallbackDriver::Stop driver=%p", Graph(), this));
TRACE("AudioCallbackDriver::Stop");
MOZ_ASSERT(OnCubebOperationThread());
cubeb_stream_register_device_changed_callback(mAudioStream, nullptr);
if (cubeb_stream_stop(mAudioStream) != CUBEB_OK) {
NS_WARNING("Could not stop cubeb stream for MTG.");
} else {
mAudioStreamState = AudioStreamState::None;
}
}
void AudioCallbackDriver::Shutdown() {
MOZ_ASSERT(NS_IsMainThread());
RefPtr<FallbackWrapper> fallback;
{
auto fallbackLock = mFallback.Lock();
fallback = fallbackLock.ref();
fallbackLock.ref() = nullptr;
}
if (fallback) {
LOG(LogLevel::Debug,
("%p: Releasing fallback driver %p.", Graph(), fallback.get()));
fallback->Shutdown();
}
LOG(LogLevel::Debug,
("%p: Releasing audio driver off main thread (GraphDriver::Shutdown).",
Graph()));
nsLiteralCString reason("AudioCallbackDriver::Shutdown");
NS_DispatchAndSpinEventLoopUntilComplete(
reason, mCubebOperationThread,
NS_NewRunnableFunction(reason.get(),
[self = RefPtr{this}] { self->Stop(); }));
}
void AudioCallbackDriver::SetStreamName(const nsACString& aStreamName) {
MOZ_ASSERT(InIteration() || !ThreadRunning());
if (aStreamName == mStreamName) {
return;
}
// Record the stream name, which will be passed onto the next driver, if
// any, either from this driver or the fallback driver.
GraphDriver::SetStreamName(aStreamName);
{
auto fallbackLock = mFallback.Lock();
FallbackWrapper* fallback = fallbackLock.ref().get();
if (fallback) {
MOZ_ASSERT(fallback->InIteration());
fallback->SetStreamName(aStreamName);
}
}
AudioStreamState streamState = mAudioStreamState;
if (streamState != AudioStreamState::None &&
streamState != AudioStreamState::Stopping) {
MOZ_ALWAYS_SUCCEEDS(mCubebOperationThread->Dispatch(
NS_NewRunnableFunction("AudioCallbackDriver SetStreamName()",
[self = RefPtr{this}, streamName = mStreamName] {
self->SetCubebStreamName(streamName);
})));
}
}
/* static */
long AudioCallbackDriver::DataCallback_s(cubeb_stream* aStream, void* aUser,
const void* aInputBuffer,
void* aOutputBuffer, long aFrames) {
AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
return driver->DataCallback(static_cast<const AudioDataValue*>(aInputBuffer),
static_cast<AudioDataValue*>(aOutputBuffer),
aFrames);
}
/* static */
void AudioCallbackDriver::StateCallback_s(cubeb_stream* aStream, void* aUser,
cubeb_state aState) {
AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
driver->StateCallback(aState);
}
/* static */
void AudioCallbackDriver::DeviceChangedCallback_s(void* aUser) {
AudioCallbackDriver* driver = reinterpret_cast<AudioCallbackDriver*>(aUser);
driver->DeviceChangedCallback();
}
AudioCallbackDriver::AutoInCallback::AutoInCallback(
AudioCallbackDriver* aDriver)
: mDriver(aDriver) {
MOZ_ASSERT(mDriver->mAudioThreadIdInCb == std::thread::id());
mDriver->mAudioThreadIdInCb = std::this_thread::get_id();
}
AudioCallbackDriver::AutoInCallback::~AutoInCallback() {
MOZ_ASSERT(mDriver->mAudioThreadIdInCb == std::this_thread::get_id());
mDriver->mAudioThreadIdInCb = std::thread::id();
}
bool AudioCallbackDriver::CheckThreadIdChanged() {
ProfilerThreadId id = profiler_current_thread_id();
if (id != mAudioThreadId) {
mAudioThreadId = id;
return true;
}
return false;
}
long AudioCallbackDriver::DataCallback(const AudioDataValue* aInputBuffer,
AudioDataValue* aOutputBuffer,
long aFrames) {
if (!mSandboxed && CheckThreadIdChanged()) {
CallbackThreadRegistry::Get()->Register(mAudioThreadId,
"NativeAudioCallback");
}
if (mAudioStreamState.compareExchange(AudioStreamState::Starting,
AudioStreamState::Running)) {
LOG(LogLevel::Verbose, ("%p: AudioCallbackDriver %p First audio callback "
"close the Fallback driver",
Graph(), this));
}
FallbackDriverState fallbackState = mFallbackDriverState;
if (MOZ_UNLIKELY(fallbackState == FallbackDriverState::Stopped)) {
// We're supposed to stop.
PodZero(aOutputBuffer, aFrames * mOutputChannelCount);
if (!mSandboxed) {
CallbackThreadRegistry::Get()->Unregister(mAudioThreadId);
}
return aFrames - 1;
}
AudioStreamState audioStreamState = mAudioStreamState;
if (MOZ_UNLIKELY(audioStreamState == AudioStreamState::ChangingDevice ||
fallbackState == FallbackDriverState::Running)) {
// Wait for the fallback driver to stop. Wake it up so it can stop if it's
// sleeping.
LOG(LogLevel::Verbose,
("%p: AudioCallbackDriver %p Waiting for the Fallback driver to stop",
Graph(), this));
EnsureNextIteration();
PodZero(aOutputBuffer, aFrames * mOutputChannelCount);
return aFrames;
}
MOZ_ASSERT(audioStreamState == AudioStreamState::Running);
TRACE_AUDIO_CALLBACK_BUDGET("AudioCallbackDriver real-time budget", aFrames,
mSampleRate);
TRACE("AudioCallbackDriver::DataCallback");
#ifdef DEBUG
AutoInCallback aic(this);
#endif
uint32_t durationMS = aFrames * 1000 / mSampleRate;
// For now, simply average the duration with the previous
// duration so there is some damping against sudden changes.
if (!mIterationDurationMS) {
mIterationDurationMS = durationMS;
} else {
mIterationDurationMS = (mIterationDurationMS * 3) + durationMS;
mIterationDurationMS /= 4;
}
mBuffer.SetBuffer(aOutputBuffer, aFrames);
// fill part or all with leftover data from last iteration (since we
// align to Audio blocks)
uint32_t alreadyBuffered = mScratchBuffer.Empty(mBuffer);
// State computed time is decided by the audio callback's buffer length. We
// compute the iteration start and end from there, trying to keep the amount
// of buffering in the graph constant.
GraphTime nextStateComputedTime =
MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(mStateComputedTime +
mBuffer.Available());
auto iterationStart = mIterationEnd;
// inGraph is the number of audio frames there is between the state time and
// the current time, i.e. the maximum theoretical length of the interval we
// could use as [iterationStart; mIterationEnd].
GraphTime inGraph = mStateComputedTime - iterationStart;
// We want the interval [iterationStart; mIterationEnd] to be before the
// interval [mStateComputedTime; nextStateComputedTime]. We also want
// the distance between these intervals to be roughly equivalent each time, to
// ensure there is no clock drift between current time and state time. Since
// we can't act on the state time because we have to fill the audio buffer, we
// reclock the current time against the state time, here.
mIterationEnd = iterationStart + 0.8 * inGraph;
LOG(LogLevel::Verbose,
("%p: interval[%ld; %ld] state[%ld; %ld] (frames: %ld) (durationMS: %u) "
"(duration ticks: %ld)",
Graph(), (long)iterationStart, (long)mIterationEnd,
(long)mStateComputedTime, (long)nextStateComputedTime, (long)aFrames,
(uint32_t)durationMS,
(long)(nextStateComputedTime - mStateComputedTime)));
if (mStateComputedTime < mIterationEnd) {
LOG(LogLevel::Error, ("%p: Media graph global underrun detected", Graph()));
MOZ_ASSERT_UNREACHABLE("We should not underrun in full duplex");
mIterationEnd = mStateComputedTime;
}
// Process mic data if any/needed
if (aInputBuffer && mInputChannelCount > 0) {
Graph()->NotifyInputData(aInputBuffer, static_cast<size_t>(aFrames),
mSampleRate, mInputChannelCount, alreadyBuffered);
}
IterationResult result =
Graph()->OneIteration(nextStateComputedTime, mIterationEnd, this);
mStateComputedTime = nextStateComputedTime;
MOZ_ASSERT(mBuffer.Available() == 0,
"The graph should have filled the buffer");
mBuffer.BufferFilled();
#ifdef MOZ_SAMPLE_TYPE_FLOAT32
// Prevent returning NaN to the OS mixer, and propagating NaN into the reverse
// stream of the AEC.
NaNToZeroInPlace(aOutputBuffer, aFrames * mOutputChannelCount);
#endif
#ifdef XP_MACOSX
// This only happens when the output is on a macbookpro's external speaker,
// that are stereo, but let's just be safe.
if (mNeedsPanning && mOutputChannelCount == 2) {
// hard pan to the right
for (uint32_t i = 0; i < aFrames * 2; i += 2) {
aOutputBuffer[i + 1] += aOutputBuffer[i];
aOutputBuffer[i] = 0.0;
}
}
#endif
// No-op if MOZ_DUMP_AUDIO is not defined as an environment variable
if (aInputBuffer) {
mInputStreamFile.Write(static_cast<const AudioDataValue*>(aInputBuffer),
aFrames * mInputChannelCount);
}
mOutputStreamFile.Write(static_cast<const AudioDataValue*>(aOutputBuffer),
aFrames * mOutputChannelCount);
if (result.IsStop()) {
if (mInputDeviceID) {
mGraphInterface->NotifyInputStopped();
}
// Signal that we have stopped.
result.Stopped();
// Update the flag before handing over the graph and going to drain.
mAudioStreamState = AudioStreamState::Stopping;
if (!mSandboxed) {
CallbackThreadRegistry::Get()->Unregister(mAudioThreadId);
}
return aFrames - 1;
}
if (GraphDriver* nextDriver = result.NextDriver()) {
LOG(LogLevel::Debug,
("%p: Switching to %s driver.", Graph(),
nextDriver->AsAudioCallbackDriver() ? "audio" : "system"));
if (mInputDeviceID) {
mGraphInterface->NotifyInputStopped();
}
result.Switched();
mAudioStreamState = AudioStreamState::Stopping;
nextDriver->SetState(mStreamName, mIterationEnd, mStateComputedTime);
nextDriver->Start();
if (!mSandboxed) {
CallbackThreadRegistry::Get()->Unregister(mAudioThreadId);
}
// Returning less than aFrames starts the draining and eventually stops the
// audio thread. This function will never get called again.
return aFrames - 1;
}
MOZ_ASSERT(result.IsStillProcessing());
return aFrames;
}
static const char* StateToString(cubeb_state aState) {
switch (aState) {
case CUBEB_STATE_STARTED:
return "STARTED";
case CUBEB_STATE_STOPPED:
return "STOPPED";
case CUBEB_STATE_DRAINED:
return "DRAINED";
case CUBEB_STATE_ERROR:
return "ERROR";
default:
MOZ_CRASH("Unexpected state!");
}
}
void AudioCallbackDriver::StateCallback(cubeb_state aState) {
MOZ_ASSERT(!InIteration());
LOG(LogLevel::Debug,
("AudioCallbackDriver(%p) State: %s", this, StateToString(aState)));
if (aState == CUBEB_STATE_STARTED || aState == CUBEB_STATE_STOPPED) {
// Nothing to do for STARTED.
//
// For STOPPED, don't reset mAudioStreamState until after
// cubeb_stream_stop() returns, as wasapi_stream_stop() dispatches
// CUBEB_STATE_STOPPED before ensuring that data callbacks have finished.
// https://searchfox.org/mozilla-central/rev/f9beb753a84aa297713d1565dcd0c5e3c66e4174/media/libcubeb/src/cubeb_wasapi.cpp#3009,3012
return;
}
AudioStreamState streamState = mAudioStreamState;
if (streamState < AudioStreamState::Starting) {
// mAudioStream has already entered STOPPED, DRAINED, or ERROR.
// Don't reset a Pending state indicating that a task to destroy
// mAudioStream and init a new cubeb_stream has already been triggered.
return;
}
// Reset for DRAINED or ERROR.
streamState = mAudioStreamState.exchange(AudioStreamState::None);
if (aState == CUBEB_STATE_ERROR) {
// About to hand over control of the graph. Do not start a new driver if
// StateCallback() receives an error for this stream while the main thread
// or another driver has control of the graph.
if (streamState == AudioStreamState::Starting ||
streamState == AudioStreamState::ChangingDevice ||
streamState == AudioStreamState::Running) {
if (mFallbackDriverState.compareExchange(FallbackDriverState::None,
FallbackDriverState::Running)) {
// Only switch to fallback if it's not already running. It could be
// running with the callback driver having started but not seen a single
// callback yet. I.e., handover from fallback to callback is not done.
if (mInputDeviceID) {
#ifdef DEBUG
// No audio callback after an error. We're calling into the graph here
// so we need to be regarded as "in iteration".
AutoInCallback aic(this);
#endif
mGraphInterface->NotifyInputStopped();
}
FallbackToSystemClockDriver();
}
}
}
}
void AudioCallbackDriver::MixerCallback(AudioChunk* aMixedBuffer,
uint32_t aSampleRate) {
MOZ_ASSERT(InIteration());
uint32_t toWrite = mBuffer.Available();
TrackTime frameCount = aMixedBuffer->mDuration;
if (!mBuffer.Available() && frameCount > 0) {
NS_WARNING("DataCallback buffer full, expect frame drops.");
}
MOZ_ASSERT(mBuffer.Available() <= frameCount);
mBuffer.WriteFrames(*aMixedBuffer, mBuffer.Available());
MOZ_ASSERT(mBuffer.Available() == 0,
"Missing frames to fill audio callback's buffer.");
if (toWrite == frameCount) {
return;
}
aMixedBuffer->SliceTo(toWrite, frameCount);
DebugOnly<uint32_t> written = mScratchBuffer.Fill(*aMixedBuffer);
NS_WARNING_ASSERTION(written == frameCount - toWrite, "Dropping frames.");
};
void AudioCallbackDriver::PanOutputIfNeeded(bool aMicrophoneActive) {
#ifdef XP_MACOSX
TRACE("AudioCallbackDriver::PanOutputIfNeeded");
cubeb_device* out = nullptr;
int rv;
char name[128];
size_t length = sizeof(name);
rv = sysctlbyname("hw.model", name, &length, NULL, 0);
if (rv) {
return;
}
int major, minor;
for (uint32_t i = 0; i < length; i++) {
// skip the model name
if (isalpha(name[i])) {
continue;
}
sscanf(name + i, "%d,%d", &major, &minor);
break;
}
enum MacbookModel { MacBook, MacBookPro, MacBookAir, NotAMacbook };
MacbookModel model;
if (!strncmp(name, "MacBookPro", length)) {
model = MacBookPro;
} else if (strncmp(name, "MacBookAir", length)) {
model = MacBookAir;
} else if (strncmp(name, "MacBook", length)) {
model = MacBook;
} else {
model = NotAMacbook;
}
// For macbook pro before 2016 model (change of chassis), hard pan the audio
// to the right if the speakers are in use to avoid feedback.
if (model == MacBookPro && major <= 12) {
if (cubeb_stream_get_current_device(mAudioStream, &out) == CUBEB_OK) {
MOZ_ASSERT(out);
// Check if we are currently outputing sound on external speakers.
if (out->output_name && !strcmp(out->output_name, "ispk")) {
// Pan everything to the right speaker.
LOG(LogLevel::Debug, ("Using the built-in speakers, with%s audio input",
aMicrophoneActive ? "" : "out"));
mNeedsPanning = aMicrophoneActive;
} else {
LOG(LogLevel::Debug, ("Using an external output device"));
mNeedsPanning = false;
}
cubeb_stream_device_destroy(mAudioStream, out);
}
}
#endif
}
void AudioCallbackDriver::DeviceChangedCallback() {
MOZ_ASSERT(!InIteration());
// Set this before the atomic write.
mChangingDeviceStartTime = TimeStamp::Now();
if (mAudioStreamState.compareExchange(AudioStreamState::Running,
AudioStreamState::ChangingDevice)) {
// Change to ChangingDevice only if we're running, i.e. there has been a
// data callback and no state callback saying otherwise.
// - If the audio stream is not running, it has either been stopped or it is
// starting. In the latter case we assume there will be no data callback
// coming until after the device change is done.
// - If the audio stream is running here, there is no guarantee from the
// cubeb mac backend that no more data callback will occur before the
// device change takes place. They will however stop *soon*, and we hope
// they stop before the first callback from the fallback driver. If the
// fallback driver callback occurs before the last data callback before
// the device switch, the worst case is that a long period of time
// (seconds) may pass without the graph getting iterated at all.
Result<bool, FallbackDriverState> res = TryStartingFallbackDriver();
LOG(LogLevel::Info,
("%p: AudioCallbackDriver %p underlying default device is changing. "
"Fallback %s.",
Graph(), this,
res.isOk() ? "started"
: (res.inspectErr() == FallbackDriverState::Running
? "already running"
: "has been stopped")));
if (res.isErr() && res.inspectErr() == FallbackDriverState::Stopped) {
mChangingDeviceStartTime = TimeStamp();
}
}
// Tell the audio engine the device has changed, it might want to reset some
// state.
Graph()->DeviceChanged();
#ifdef XP_MACOSX
RefPtr<AudioCallbackDriver> self(this);
bool hasInput = mInputChannelCount;
NS_DispatchBackgroundTask(NS_NewRunnableFunction(
"PanOutputIfNeeded", [self{std::move(self)}, hasInput]() {
self->PanOutputIfNeeded(hasInput);
}));
#endif
}
uint32_t AudioCallbackDriver::IterationDuration() {
MOZ_ASSERT(InIteration());
// The real fix would be to have an API in cubeb to give us the number. Short
// of that, we approximate it here. bug 1019507
return mIterationDurationMS;
}
void AudioCallbackDriver::EnsureNextIteration() {
if (mFallbackDriverState == FallbackDriverState::Running) {
auto fallback = mFallback.Lock();
if (fallback.ref()) {
fallback.ref()->EnsureNextIteration();
}
}
}
TimeDuration AudioCallbackDriver::AudioOutputLatency() {
TRACE("AudioCallbackDriver::AudioOutputLatency");
uint32_t latencyFrames;
int rv = cubeb_stream_get_latency(mAudioStream, &latencyFrames);
if (rv || mSampleRate == 0) {
return TimeDuration::FromSeconds(0.0);
}
return TimeDuration::FromSeconds(static_cast<double>(latencyFrames) /
mSampleRate);
}
bool AudioCallbackDriver::HasFallback() const {
MOZ_ASSERT(InIteration());
return mFallbackDriverState != FallbackDriverState::None;
}
bool AudioCallbackDriver::OnFallback() const {
MOZ_ASSERT(InIteration());
return mFallbackDriverState == FallbackDriverState::Running;
}
Result<bool, AudioCallbackDriver::FallbackDriverState>
AudioCallbackDriver::TryStartingFallbackDriver() {
FallbackDriverState oldState =
mFallbackDriverState.exchange(FallbackDriverState::Running);
switch (oldState) {
case FallbackDriverState::None:
// None -> Running: we can start the fallback.
FallbackToSystemClockDriver();
return true;
case FallbackDriverState::Stopped:
// Stopped -> Running: Invalid edge, the graph has told us to stop.
// Restore the state.
mFallbackDriverState = oldState;
[[fallthrough]];
case FallbackDriverState::Running:
// Nothing to do, return the state.
return Err(oldState);
}
MOZ_CRASH("Unexpected fallback state");
}
void AudioCallbackDriver::FallbackToSystemClockDriver() {
MOZ_ASSERT(mFallbackDriverState == FallbackDriverState::Running);
DebugOnly<AudioStreamState> audioStreamState =
static_cast<AudioStreamState>(mAudioStreamState);
MOZ_ASSERT(audioStreamState == AudioStreamState::None ||
audioStreamState == AudioStreamState::Pending ||
audioStreamState == AudioStreamState::ChangingDevice);
LOG(LogLevel::Debug,
("%p: AudioCallbackDriver %p Falling back to SystemClockDriver.", Graph(),
this));
mNextReInitBackoffStep =
TimeDuration::FromMilliseconds(AUDIO_INITIAL_FALLBACK_BACKOFF_STEP_MS);
mNextReInitAttempt = TimeStamp::Now() + mNextReInitBackoffStep;
auto fallback =
MakeRefPtr<FallbackWrapper>(Graph(), this, mSampleRate, mStreamName,
mIterationEnd, mStateComputedTime);
{
auto driver = mFallback.Lock();
MOZ_RELEASE_ASSERT(!driver.ref());
driver.ref() = fallback;
}
fallback->Start();
}
void AudioCallbackDriver::FallbackDriverStopped(GraphTime aIterationEnd,
GraphTime aStateComputedTime,
FallbackDriverState aState) {
LOG(LogLevel::Debug,
("%p: AudioCallbackDriver %p Fallback driver has stopped.", Graph(),
this));
mIterationEnd = aIterationEnd;
mStateComputedTime = aStateComputedTime;
mNextReInitAttempt = TimeStamp();
mNextReInitBackoffStep = TimeDuration();
{
auto fallback = mFallback.Lock();
MOZ_ASSERT(fallback.ref()->OnThread());
fallback.ref() = nullptr;
}
MOZ_ASSERT(aState == FallbackDriverState::None ||
aState == FallbackDriverState::Stopped);
mFallbackDriverState = aState;
AudioStreamState audioState = mAudioStreamState;
LOG(LogLevel::Debug,
("%p: AudioCallbackDriver %p Fallback driver stopped.%s%s", Graph(), this,
aState == FallbackDriverState::Stopped ? " Draining." : "",
aState == FallbackDriverState::None &&
audioState == AudioStreamState::ChangingDevice
? " Starting another due to device change."
: ""));
if (aState == FallbackDriverState::None) {
MOZ_ASSERT(audioState == AudioStreamState::Running ||
audioState == AudioStreamState::ChangingDevice);
if (audioState == AudioStreamState::ChangingDevice) {
MOZ_ALWAYS_OK(TryStartingFallbackDriver());
}
}
}
void AudioCallbackDriver::MaybeStartAudioStream() {
AudioStreamState streamState = mAudioStreamState;
if (streamState != AudioStreamState::None) {
LOG(LogLevel::Verbose,
("%p: AudioCallbackDriver %p Cannot re-init.", Graph(), this));
return;
}
TimeStamp now = TimeStamp::Now();
if (now < mNextReInitAttempt) {
LOG(LogLevel::Verbose,
("%p: AudioCallbackDriver %p Not time to re-init yet. %.3fs left.",
Graph(), this, (mNextReInitAttempt - now).ToSeconds()));
return;
}
LOG(LogLevel::Debug, ("%p: AudioCallbackDriver %p Attempting to re-init "
"audio stream from fallback driver.",
Graph(), this));
mNextReInitBackoffStep =
std::min(mNextReInitBackoffStep * 2,
TimeDuration::FromMilliseconds(
StaticPrefs::media_audio_device_retry_ms()));
mNextReInitAttempt = now + mNextReInitBackoffStep;
Start();
}
cubeb_input_processing_params
AudioCallbackDriver::RequestedInputProcessingParams() const {
MOZ_ASSERT(InIteration());
return mRequestedInputProcessingParams;
}
void AudioCallbackDriver::SetRequestedInputProcessingParams(
cubeb_input_processing_params aParams) {
MOZ_ASSERT(InIteration());
if (mRequestedInputProcessingParams == aParams) {
return;
}
LOG(LogLevel::Info,
("AudioCallbackDriver %p, Input processing params %s requested.", this,
CubebUtils::ProcessingParamsToString(aParams).get()));
mRequestedInputProcessingParams = aParams;
MOZ_ALWAYS_SUCCEEDS(mCubebOperationThread->Dispatch(
NS_NewRunnableFunction(__func__, [this, self = RefPtr(this), aParams] {
SetInputProcessingParams(aParams);
})));
}
void AudioCallbackDriver::SetInputProcessingParams(
cubeb_input_processing_params aParams) {
MOZ_ASSERT(OnCubebOperationThread());
auto requested = aParams;
auto result = ([&]() -> Maybe<Result<cubeb_input_processing_params, int>> {
// This function decides how to handle the request.
// Returning Nothing() does nothing, because either
// 1) there is no update since the previous state, or
// 2) handling is deferred to a later time.
// Returning Some() result will forward that result to
// AudioDataListener::OnInputProcessingParamsResult on the callback
// thread.
if (!mAudioStream) {
// No Init yet.
LOG(LogLevel::Debug, ("AudioCallbackDriver %p, has no cubeb stream to "
"set processing params on!",
this));
return Nothing();
}
if (mAudioStreamState == AudioStreamState::None) {
// Driver (and cubeb stream) was stopped.
return Nothing();
}
cubeb_input_processing_params supported;
auto handle = CubebUtils::GetCubeb();
int r = cubeb_get_supported_input_processing_params(handle->Context(),
&supported);
if (r != CUBEB_OK) {
LOG(LogLevel::Debug,
("AudioCallbackDriver %p, no supported processing params", this));
return Some(Err(CUBEB_ERROR_NOT_SUPPORTED));
}
aParams &= supported;
LOG(LogLevel::Debug,
("AudioCallbackDriver %p, requested processing params %s reduced to %s "
"by supported params %s",
this, CubebUtils::ProcessingParamsToString(requested).get(),
CubebUtils::ProcessingParamsToString(aParams).get(),
CubebUtils::ProcessingParamsToString(supported).get()));
if (aParams == mConfiguredInputProcessingParams) {
LOG(LogLevel::Debug,
("AudioCallbackDriver %p, no change in processing params %s. Not "
"attempting reconfiguration.",
this, CubebUtils::ProcessingParamsToString(aParams).get()));
return Some(aParams);
}
mConfiguredInputProcessingParams = aParams;
r = cubeb_stream_set_input_processing_params(mAudioStream, aParams);
if (r == CUBEB_OK) {
LOG(LogLevel::Info,
("AudioCallbackDriver %p, input processing params set to %s", this,
CubebUtils::ProcessingParamsToString(aParams).get()));
return Some(aParams);
}
LOG(LogLevel::Info,
("AudioCallbackDriver %p, failed setting input processing params to "
"%s. r=%d",
this, CubebUtils::ProcessingParamsToString(aParams).get(), r));
return Some(Err(r));
})();
if (!result) {
return;
}
MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(
NS_NewRunnableFunction(__func__, [this, self = RefPtr(this), requested,
result = result.extract()]() mutable {
LOG(LogLevel::Debug,
("AudioCallbackDriver %p, Notifying of input processing params %s. "
"r=%d",
this,
CubebUtils::ProcessingParamsToString(
result.unwrapOr(CUBEB_INPUT_PROCESSING_PARAM_NONE))
.get(),
result.isErr() ? result.inspectErr() : CUBEB_OK));
mGraphInterface->NotifySetRequestedInputProcessingParamsResult(
this, requested, std::move(result));
})));
}
} // namespace mozilla
// avoid redefined macro in unified build
#undef LOG
|