1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
|
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef DriftCompensation_h_
#define DriftCompensation_h_
#include "MediaSegment.h"
#include "VideoUtils.h"
#include "mozilla/Atomics.h"
#include "mozilla/Unused.h"
namespace mozilla {
static LazyLogModule gDriftCompensatorLog("DriftCompensator");
#define LOG(type, ...) MOZ_LOG(gDriftCompensatorLog, type, (__VA_ARGS__))
/**
* DriftCompensator can be used to handle drift between audio and video tracks
* from the MediaTrackGraph.
*
* Drift can occur because audio is driven by a MediaTrackGraph running off an
* audio callback, thus it's progressed by the clock of one the audio output
* devices on the user's machine. Video on the other hand is always expressed in
* wall-clock TimeStamps, i.e., it's progressed by the system clock. These
* clocks will, over time, drift apart.
*
* Do not use the DriftCompensator across multiple audio tracks, as it will
* automatically record the start time of the first audio samples, and all
* samples for the same audio track on the same audio clock will have to be
* processed to retain accuracy.
*
* DriftCompensator is designed to be used from two threads:
* - The audio thread for notifications of audio samples.
* - The video thread for compensating drift of video frames to match the audio
* clock.
*/
class DriftCompensator {
const RefPtr<nsIEventTarget> mVideoThread;
const TrackRate mAudioRate;
// Number of audio samples produced. Any thread.
Atomic<TrackTime> mAudioSamples{0};
// Time the first audio samples were added. mVideoThread only.
TimeStamp mAudioStartTime;
void SetAudioStartTime(TimeStamp aTime) {
MOZ_ASSERT(mVideoThread->IsOnCurrentThread());
MOZ_ASSERT(mAudioStartTime.IsNull());
mAudioStartTime = aTime;
}
protected:
virtual ~DriftCompensator() = default;
public:
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DriftCompensator)
DriftCompensator(RefPtr<nsIEventTarget> aVideoThread, TrackRate aAudioRate)
: mVideoThread(std::move(aVideoThread)), mAudioRate(aAudioRate) {
MOZ_ASSERT(mAudioRate > 0);
}
void NotifyAudioStart(TimeStamp aStart) {
MOZ_ASSERT(mAudioSamples == 0);
LOG(LogLevel::Info, "DriftCompensator %p at rate %d started", this,
mAudioRate);
nsresult rv = mVideoThread->Dispatch(NewRunnableMethod<TimeStamp>(
"DriftCompensator::SetAudioStartTime", this,
&DriftCompensator::SetAudioStartTime, aStart));
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
Unused << rv;
}
/**
* aSamples is the number of samples fed by an AudioStream.
*/
void NotifyAudio(TrackTime aSamples) {
MOZ_ASSERT(aSamples > 0);
mAudioSamples += aSamples;
LOG(LogLevel::Verbose,
"DriftCompensator %p Processed another %" PRId64
" samples; now %.3fs audio",
this, aSamples, static_cast<double>(mAudioSamples) / mAudioRate);
}
/**
* Drift compensates a video TimeStamp based on historical audio data.
*/
virtual TimeStamp GetVideoTime(TimeStamp aNow, TimeStamp aTime) {
MOZ_ASSERT(mVideoThread->IsOnCurrentThread());
TrackTime samples = mAudioSamples;
if (samples / mAudioRate < 10) {
// We don't apply compensation for the first 10 seconds because of the
// higher inaccuracy during this time.
LOG(LogLevel::Debug, "DriftCompensator %p %" PRId64 "ms so far; ignoring",
this, samples * 1000 / mAudioRate);
return aTime;
}
if (aNow == mAudioStartTime) {
LOG(LogLevel::Warning,
"DriftCompensator %p video scale 0, assuming no drift", this);
return aTime;
}
double videoScaleUs = (aNow - mAudioStartTime).ToMicroseconds();
double audioScaleUs = FramesToUsecs(samples, mAudioRate).value();
double videoDurationUs = (aTime - mAudioStartTime).ToMicroseconds();
TimeStamp reclocked =
mAudioStartTime + TimeDuration::FromMicroseconds(
videoDurationUs * audioScaleUs / videoScaleUs);
LOG(LogLevel::Debug,
"DriftCompensator %p GetVideoTime, v-now: %.3fs, a-now: %.3fs; %.3fs "
"-> %.3fs (d %.3fms)",
this, (aNow - mAudioStartTime).ToSeconds(),
TimeDuration::FromMicroseconds(audioScaleUs).ToSeconds(),
(aTime - mAudioStartTime).ToSeconds(),
(reclocked - mAudioStartTime).ToSeconds(),
(reclocked - aTime).ToMilliseconds());
return reclocked;
}
};
#undef LOG
} // namespace mozilla
#endif /* DriftCompensation_h_ */
|
