1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
|
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/rtp_vp8_ref_finder.h"
#include <utility>
#include "rtc_base/logging.h"
namespace webrtc {
RtpFrameReferenceFinder::ReturnVector RtpVp8RefFinder::ManageFrame(
std::unique_ptr<RtpFrameObject> frame) {
const RTPVideoHeaderVP8& codec_header = absl::get<RTPVideoHeaderVP8>(
frame->GetRtpVideoHeader().video_type_header);
if (codec_header.temporalIdx != kNoTemporalIdx)
frame->SetTemporalIndex(codec_header.temporalIdx);
int64_t unwrapped_tl0 = tl0_unwrapper_.Unwrap(codec_header.tl0PicIdx & 0xFF);
FrameDecision decision =
ManageFrameInternal(frame.get(), codec_header, unwrapped_tl0);
RtpFrameReferenceFinder::ReturnVector res;
switch (decision) {
case kStash:
if (stashed_frames_.size() > kMaxStashedFrames) {
stashed_frames_.pop_back();
}
stashed_frames_.push_front(
{.unwrapped_tl0 = unwrapped_tl0, .frame = std::move(frame)});
return res;
case kHandOff:
res.push_back(std::move(frame));
RetryStashedFrames(res);
return res;
case kDrop:
return res;
}
return res;
}
RtpVp8RefFinder::FrameDecision RtpVp8RefFinder::ManageFrameInternal(
RtpFrameObject* frame,
const RTPVideoHeaderVP8& codec_header,
int64_t unwrapped_tl0) {
// Protect against corrupted packets with arbitrary large temporal idx.
if (codec_header.temporalIdx >= kMaxTemporalLayers)
return kDrop;
frame->SetSpatialIndex(0);
frame->SetId(codec_header.pictureId & 0x7FFF);
if (last_picture_id_ == -1)
last_picture_id_ = frame->Id();
// Clean up info about not yet received frames that are too old.
uint16_t old_picture_id =
Subtract<kFrameIdLength>(frame->Id(), kMaxNotYetReceivedFrames);
auto clean_frames_to = not_yet_received_frames_.lower_bound(old_picture_id);
not_yet_received_frames_.erase(not_yet_received_frames_.begin(),
clean_frames_to);
// Avoid re-adding picture ids that were just erased.
if (AheadOf<uint16_t, kFrameIdLength>(old_picture_id, last_picture_id_)) {
last_picture_id_ = old_picture_id;
}
// Find if there has been a gap in fully received frames and save the picture
// id of those frames in `not_yet_received_frames_`.
if (AheadOf<uint16_t, kFrameIdLength>(frame->Id(), last_picture_id_)) {
do {
last_picture_id_ = Add<kFrameIdLength>(last_picture_id_, 1);
not_yet_received_frames_.insert(last_picture_id_);
} while (last_picture_id_ != frame->Id());
}
// Clean up info for base layers that are too old.
int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxLayerInfo;
auto clean_layer_info_to = layer_info_.lower_bound(old_tl0_pic_idx);
layer_info_.erase(layer_info_.begin(), clean_layer_info_to);
if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
if (codec_header.temporalIdx != 0) {
return kDrop;
}
frame->num_references = 0;
layer_info_[unwrapped_tl0].fill(-1);
UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
return kHandOff;
}
auto layer_info_it = layer_info_.find(
codec_header.temporalIdx == 0 ? unwrapped_tl0 - 1 : unwrapped_tl0);
// If we don't have the base layer frame yet, stash this frame.
if (layer_info_it == layer_info_.end())
return kStash;
// A non keyframe base layer frame has been received, copy the layer info
// from the previous base layer frame and set a reference to the previous
// base layer frame.
if (codec_header.temporalIdx == 0) {
layer_info_it =
layer_info_.emplace(unwrapped_tl0, layer_info_it->second).first;
frame->num_references = 1;
int64_t last_pid_on_layer = layer_info_it->second[0];
// Is this an old frame that has already been used to update the state? If
// so, drop it.
if (AheadOrAt<uint16_t, kFrameIdLength>(last_pid_on_layer, frame->Id())) {
return kDrop;
}
frame->references[0] = last_pid_on_layer;
UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
return kHandOff;
}
// Layer sync frame, this frame only references its base layer frame.
if (codec_header.layerSync) {
frame->num_references = 1;
int64_t last_pid_on_layer = layer_info_it->second[codec_header.temporalIdx];
// Is this an old frame that has already been used to update the state? If
// so, drop it.
if (last_pid_on_layer != -1 &&
AheadOrAt<uint16_t, kFrameIdLength>(last_pid_on_layer, frame->Id())) {
return kDrop;
}
frame->references[0] = layer_info_it->second[0];
UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
return kHandOff;
}
// Find all references for this frame.
frame->num_references = 0;
for (uint8_t layer = 0; layer <= codec_header.temporalIdx; ++layer) {
// If we have not yet received a previous frame on this temporal layer,
// stash this frame.
if (layer_info_it->second[layer] == -1)
return kStash;
// If the last frame on this layer is ahead of this frame it means that
// a layer sync frame has been received after this frame for the same
// base layer frame, drop this frame.
if (AheadOf<uint16_t, kFrameIdLength>(layer_info_it->second[layer],
frame->Id())) {
return kDrop;
}
// If we have not yet received a frame between this frame and the referenced
// frame then we have to wait for that frame to be completed first.
auto not_received_frame_it =
not_yet_received_frames_.upper_bound(layer_info_it->second[layer]);
if (not_received_frame_it != not_yet_received_frames_.end() &&
AheadOf<uint16_t, kFrameIdLength>(frame->Id(),
*not_received_frame_it)) {
return kStash;
}
if (!(AheadOf<uint16_t, kFrameIdLength>(frame->Id(),
layer_info_it->second[layer]))) {
RTC_LOG(LS_WARNING) << "Frame with picture id " << frame->Id()
<< " and packet range [" << frame->first_seq_num()
<< ", " << frame->last_seq_num()
<< "] already received, "
" dropping frame.";
return kDrop;
}
++frame->num_references;
frame->references[layer] = layer_info_it->second[layer];
}
UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
return kHandOff;
}
void RtpVp8RefFinder::UpdateLayerInfoVp8(RtpFrameObject* frame,
int64_t unwrapped_tl0,
uint8_t temporal_idx) {
auto layer_info_it = layer_info_.find(unwrapped_tl0);
// Update this layer info and newer.
while (layer_info_it != layer_info_.end()) {
if (layer_info_it->second[temporal_idx] != -1 &&
AheadOf<uint16_t, kFrameIdLength>(layer_info_it->second[temporal_idx],
frame->Id())) {
// The frame was not newer, then no subsequent layer info have to be
// update.
break;
}
layer_info_it->second[temporal_idx] = frame->Id();
++unwrapped_tl0;
layer_info_it = layer_info_.find(unwrapped_tl0);
}
not_yet_received_frames_.erase(frame->Id());
UnwrapPictureIds(frame);
}
void RtpVp8RefFinder::RetryStashedFrames(
RtpFrameReferenceFinder::ReturnVector& res) {
bool complete_frame = false;
do {
complete_frame = false;
for (auto it = stashed_frames_.begin(); it != stashed_frames_.end();) {
const RTPVideoHeaderVP8& codec_header = absl::get<RTPVideoHeaderVP8>(
it->frame->GetRtpVideoHeader().video_type_header);
FrameDecision decision =
ManageFrameInternal(it->frame.get(), codec_header, it->unwrapped_tl0);
switch (decision) {
case kStash:
++it;
break;
case kHandOff:
complete_frame = true;
res.push_back(std::move(it->frame));
[[fallthrough]];
case kDrop:
it = stashed_frames_.erase(it);
}
}
} while (complete_frame);
}
void RtpVp8RefFinder::UnwrapPictureIds(RtpFrameObject* frame) {
for (size_t i = 0; i < frame->num_references; ++i)
frame->references[i] = unwrapper_.Unwrap(frame->references[i]);
frame->SetId(unwrapper_.Unwrap(frame->Id()));
}
void RtpVp8RefFinder::ClearTo(uint16_t seq_num) {
auto it = stashed_frames_.begin();
while (it != stashed_frames_.end()) {
if (AheadOf<uint16_t>(seq_num, it->frame->first_seq_num())) {
it = stashed_frames_.erase(it);
} else {
++it;
}
}
}
} // namespace webrtc
|
