summaryrefslogtreecommitdiffstats
path: root/third_party/libwebrtc/video/video_stream_encoder_unittest.cc
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
commit36d22d82aa202bb199967e9512281e9a53db42c9 (patch)
tree105e8c98ddea1c1e4784a60a5a6410fa416be2de /third_party/libwebrtc/video/video_stream_encoder_unittest.cc
parentInitial commit. (diff)
downloadfirefox-esr-upstream.tar.xz
firefox-esr-upstream.zip
Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/libwebrtc/video/video_stream_encoder_unittest.cc')
-rw-r--r--third_party/libwebrtc/video/video_stream_encoder_unittest.cc9528
1 files changed, 9528 insertions, 0 deletions
diff --git a/third_party/libwebrtc/video/video_stream_encoder_unittest.cc b/third_party/libwebrtc/video/video_stream_encoder_unittest.cc
new file mode 100644
index 0000000000..cdd4c75ab7
--- /dev/null
+++ b/third_party/libwebrtc/video/video_stream_encoder_unittest.cc
@@ -0,0 +1,9528 @@
+
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "video/video_stream_encoder.h"
+
+#include <algorithm>
+#include <limits>
+#include <memory>
+#include <tuple>
+#include <utility>
+
+#include "absl/memory/memory.h"
+#include "api/field_trials_view.h"
+#include "api/rtp_parameters.h"
+#include "api/task_queue/default_task_queue_factory.h"
+#include "api/task_queue/task_queue_base.h"
+#include "api/task_queue/task_queue_factory.h"
+#include "api/test/mock_fec_controller_override.h"
+#include "api/test/mock_video_encoder.h"
+#include "api/test/mock_video_encoder_factory.h"
+#include "api/units/data_rate.h"
+#include "api/units/time_delta.h"
+#include "api/video/builtin_video_bitrate_allocator_factory.h"
+#include "api/video/i420_buffer.h"
+#include "api/video/nv12_buffer.h"
+#include "api/video/video_adaptation_reason.h"
+#include "api/video/video_bitrate_allocation.h"
+#include "api/video_codecs/sdp_video_format.h"
+#include "api/video_codecs/video_codec.h"
+#include "api/video_codecs/video_encoder.h"
+#include "api/video_codecs/vp8_temporal_layers.h"
+#include "api/video_codecs/vp8_temporal_layers_factory.h"
+#include "call/adaptation/test/fake_adaptation_constraint.h"
+#include "call/adaptation/test/fake_resource.h"
+#include "common_video/h264/h264_common.h"
+#include "common_video/include/video_frame_buffer.h"
+#include "media/base/video_adapter.h"
+#include "media/engine/webrtc_video_engine.h"
+#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
+#include "modules/video_coding/codecs/h264/include/h264.h"
+#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
+#include "modules/video_coding/codecs/vp8/include/vp8.h"
+#include "modules/video_coding/codecs/vp9/include/vp9.h"
+#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
+#include "modules/video_coding/codecs/vp9/svc_config.h"
+#include "modules/video_coding/utility/quality_scaler.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
+#include "modules/video_coding/utility/vp8_constants.h"
+#include "rtc_base/event.h"
+#include "rtc_base/experiments/encoder_info_settings.h"
+#include "rtc_base/gunit.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/ref_counted_object.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "system_wrappers/include/metrics.h"
+#include "test/encoder_settings.h"
+#include "test/fake_encoder.h"
+#include "test/frame_forwarder.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/mappable_native_buffer.h"
+#include "test/scoped_key_value_config.h"
+#include "test/time_controller/simulated_time_controller.h"
+#include "test/video_encoder_nullable_proxy_factory.h"
+#include "test/video_encoder_proxy_factory.h"
+#include "video/config/encoder_stream_factory.h"
+#include "video/frame_cadence_adapter.h"
+#include "video/send_statistics_proxy.h"
+
+namespace webrtc {
+
+using ::testing::_;
+using ::testing::AllOf;
+using ::testing::Eq;
+using ::testing::Field;
+using ::testing::Ge;
+using ::testing::Gt;
+using ::testing::Invoke;
+using ::testing::Le;
+using ::testing::Lt;
+using ::testing::Matcher;
+using ::testing::Mock;
+using ::testing::NiceMock;
+using ::testing::Optional;
+using ::testing::Return;
+using ::testing::SizeIs;
+using ::testing::StrictMock;
+
+namespace {
+const int kMinPixelsPerFrame = 320 * 180;
+const int kQpLow = 1;
+const int kQpHigh = 2;
+const int kMinFramerateFps = 2;
+const int kMinBalancedFramerateFps = 7;
+constexpr TimeDelta kFrameTimeout = TimeDelta::Millis(100);
+const size_t kMaxPayloadLength = 1440;
+const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
+const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
+const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
+const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
+const int kMaxInitialFramedrop = 4;
+const int kDefaultFramerate = 30;
+const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
+const int64_t kProcessIntervalMs = 1000;
+const VideoEncoder::ResolutionBitrateLimits
+ kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
+const VideoEncoder::ResolutionBitrateLimits
+ kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
+
+uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
+ 0x00, 0x00, 0x03, 0x03, 0xF4,
+ 0x05, 0x03, 0xC7, 0xE0, 0x1B,
+ 0x41, 0x10, 0x8D, 0x00};
+
+const uint8_t kCodedFrameVp8Qp25[] = {
+ 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
+ 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
+ 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
+
+VideoFrame CreateSimpleNV12Frame() {
+ return VideoFrame::Builder()
+ .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
+ /*width=*/16, /*height=*/16))
+ .build();
+}
+
+void PassAFrame(
+ TaskQueueBase* encoder_queue,
+ FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
+ int64_t ntp_time_ms) {
+ encoder_queue->PostTask([video_stream_encoder_callback, ntp_time_ms] {
+ video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms), 1,
+ CreateSimpleNV12Frame());
+ });
+}
+
+class TestBuffer : public webrtc::I420Buffer {
+ public:
+ TestBuffer(rtc::Event* event, int width, int height)
+ : I420Buffer(width, height), event_(event) {}
+
+ private:
+ friend class rtc::RefCountedObject<TestBuffer>;
+ ~TestBuffer() override {
+ if (event_)
+ event_->Set();
+ }
+ rtc::Event* const event_;
+};
+
+// A fake native buffer that can't be converted to I420. Upon scaling, it
+// produces another FakeNativeBuffer.
+class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
+ public:
+ FakeNativeBuffer(rtc::Event* event, int width, int height)
+ : event_(event), width_(width), height_(height) {}
+ webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
+ int width() const override { return width_; }
+ int height() const override { return height_; }
+ rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
+ return nullptr;
+ }
+ rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
+ int offset_x,
+ int offset_y,
+ int crop_width,
+ int crop_height,
+ int scaled_width,
+ int scaled_height) override {
+ return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
+ scaled_height);
+ }
+
+ private:
+ friend class rtc::RefCountedObject<FakeNativeBuffer>;
+ ~FakeNativeBuffer() override {
+ if (event_)
+ event_->Set();
+ }
+ rtc::Event* const event_;
+ const int width_;
+ const int height_;
+};
+
+// A fake native buffer that is backed by an NV12 buffer.
+class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
+ public:
+ FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
+ : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
+
+ webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
+ int width() const override { return nv12_buffer_->width(); }
+ int height() const override { return nv12_buffer_->height(); }
+ rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
+ return nv12_buffer_->ToI420();
+ }
+ rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
+ rtc::ArrayView<VideoFrameBuffer::Type> types) override {
+ if (absl::c_find(types, Type::kNV12) != types.end()) {
+ return nv12_buffer_;
+ }
+ return nullptr;
+ }
+ const NV12BufferInterface* GetNV12() const { return nv12_buffer_.get(); }
+
+ private:
+ friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
+ ~FakeNV12NativeBuffer() override {
+ if (event_)
+ event_->Set();
+ }
+ rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
+ rtc::Event* const event_;
+};
+
+class CpuOveruseDetectorProxy : public OveruseFrameDetector {
+ public:
+ CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer,
+ const FieldTrialsView& field_trials)
+ : OveruseFrameDetector(metrics_observer, field_trials),
+ last_target_framerate_fps_(-1),
+ framerate_updated_event_(true /* manual_reset */,
+ false /* initially_signaled */) {}
+ virtual ~CpuOveruseDetectorProxy() {}
+
+ void OnTargetFramerateUpdated(int framerate_fps) override {
+ MutexLock lock(&lock_);
+ last_target_framerate_fps_ = framerate_fps;
+ OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
+ framerate_updated_event_.Set();
+ }
+
+ int GetLastTargetFramerate() {
+ MutexLock lock(&lock_);
+ return last_target_framerate_fps_;
+ }
+
+ CpuOveruseOptions GetOptions() { return options_; }
+
+ rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
+
+ private:
+ Mutex lock_;
+ int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
+ rtc::Event framerate_updated_event_;
+};
+
+class FakeVideoSourceRestrictionsListener
+ : public VideoSourceRestrictionsListener {
+ public:
+ FakeVideoSourceRestrictionsListener()
+ : was_restrictions_updated_(false), restrictions_updated_event_() {}
+ ~FakeVideoSourceRestrictionsListener() override {
+ RTC_DCHECK(was_restrictions_updated_);
+ }
+
+ rtc::Event* restrictions_updated_event() {
+ return &restrictions_updated_event_;
+ }
+
+ // VideoSourceRestrictionsListener implementation.
+ void OnVideoSourceRestrictionsUpdated(
+ VideoSourceRestrictions restrictions,
+ const VideoAdaptationCounters& adaptation_counters,
+ rtc::scoped_refptr<Resource> reason,
+ const VideoSourceRestrictions& unfiltered_restrictions) override {
+ was_restrictions_updated_ = true;
+ restrictions_updated_event_.Set();
+ }
+
+ private:
+ bool was_restrictions_updated_;
+ rtc::Event restrictions_updated_event_;
+};
+
+auto WantsFps(Matcher<int> fps_matcher) {
+ return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
+ fps_matcher);
+}
+
+auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
+ return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
+ AllOf(max_pixel_matcher, Gt(0)));
+}
+
+auto ResolutionMax() {
+ return AllOf(
+ WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
+ Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
+ Eq(absl::nullopt)));
+}
+
+auto FpsMax() {
+ return WantsFps(Eq(kDefaultFramerate));
+}
+
+auto FpsUnlimited() {
+ return WantsFps(Eq(std::numeric_limits<int>::max()));
+}
+
+auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
+ return AllOf(WantsFps(fps_matcher), ResolutionMax());
+}
+
+auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
+ return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
+}
+
+auto FpsMaxResolutionMax() {
+ return AllOf(FpsMax(), ResolutionMax());
+}
+
+auto UnlimitedSinkWants() {
+ return AllOf(FpsUnlimited(), ResolutionMax());
+}
+
+auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
+ Matcher<int> fps_range_matcher;
+
+ if (last_frame_pixels <= 320 * 240) {
+ fps_range_matcher = AllOf(Ge(7), Le(10));
+ } else if (last_frame_pixels <= 480 * 360) {
+ fps_range_matcher = AllOf(Ge(10), Le(15));
+ } else if (last_frame_pixels <= 640 * 480) {
+ fps_range_matcher = Ge(15);
+ } else {
+ fps_range_matcher = Eq(kDefaultFramerate);
+ }
+ return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
+ fps_range_matcher);
+}
+
+auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
+ return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
+ WantsMaxPixels(Eq(other_wants.max_pixel_count)));
+}
+
+auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
+ return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
+}
+
+auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
+ return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
+}
+
+auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
+ return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
+ WantsMaxPixels(Eq(other_wants.max_pixel_count)));
+}
+
+auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
+ return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
+ WantsMaxPixels(Eq(other_wants.max_pixel_count)));
+}
+
+auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
+ return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
+ WantsMaxPixels(Lt(other_wants.max_pixel_count)));
+}
+
+auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
+ return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
+ WantsMaxPixels(Gt(other_wants.max_pixel_count)));
+}
+
+class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
+ public:
+ VideoStreamEncoderUnderTest(
+ TimeController* time_controller,
+ std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
+ std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
+ encoder_queue,
+ SendStatisticsProxy* stats_proxy,
+ const VideoStreamEncoderSettings& settings,
+ VideoStreamEncoder::BitrateAllocationCallbackType
+ allocation_callback_type,
+ const FieldTrialsView& field_trials,
+ int num_cores)
+ : VideoStreamEncoder(
+ time_controller->GetClock(),
+ num_cores,
+ stats_proxy,
+ settings,
+ std::unique_ptr<OveruseFrameDetector>(
+ overuse_detector_proxy_ =
+ new CpuOveruseDetectorProxy(stats_proxy, field_trials)),
+ std::move(cadence_adapter),
+ std::move(encoder_queue),
+ allocation_callback_type,
+ field_trials),
+ time_controller_(time_controller),
+ fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
+ fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
+ fake_adaptation_constraint_("FakeAdaptationConstraint") {
+ InjectAdaptationResource(fake_quality_resource_,
+ VideoAdaptationReason::kQuality);
+ InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
+ InjectAdaptationConstraint(&fake_adaptation_constraint_);
+ }
+
+ void SetSourceAndWaitForRestrictionsUpdated(
+ rtc::VideoSourceInterface<VideoFrame>* source,
+ const DegradationPreference& degradation_preference) {
+ FakeVideoSourceRestrictionsListener listener;
+ AddRestrictionsListenerForTesting(&listener);
+ SetSource(source, degradation_preference);
+ listener.restrictions_updated_event()->Wait(TimeDelta::Seconds(5));
+ RemoveRestrictionsListenerForTesting(&listener);
+ }
+
+ void SetSourceAndWaitForFramerateUpdated(
+ rtc::VideoSourceInterface<VideoFrame>* source,
+ const DegradationPreference& degradation_preference) {
+ overuse_detector_proxy_->framerate_updated_event()->Reset();
+ SetSource(source, degradation_preference);
+ overuse_detector_proxy_->framerate_updated_event()->Wait(
+ TimeDelta::Seconds(5));
+ }
+
+ void OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate target_bitrate,
+ DataRate stable_target_bitrate,
+ DataRate link_allocation,
+ uint8_t fraction_lost,
+ int64_t round_trip_time_ms,
+ double cwnd_reduce_ratio) {
+ OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
+ fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
+ // Bitrate is updated on the encoder queue.
+ WaitUntilTaskQueueIsIdle();
+ }
+
+ // This is used as a synchronisation mechanism, to make sure that the
+ // encoder queue is not blocked before we start sending it frames.
+ void WaitUntilTaskQueueIsIdle() {
+ time_controller_->AdvanceTime(TimeDelta::Zero());
+ }
+
+ // Triggers resource usage measurements on the fake CPU resource.
+ void TriggerCpuOveruse() {
+ rtc::Event event;
+ encoder_queue()->PostTask([this, &event] {
+ fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
+ event.Set();
+ });
+ ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
+ time_controller_->AdvanceTime(TimeDelta::Zero());
+ }
+
+ void TriggerCpuUnderuse() {
+ rtc::Event event;
+ encoder_queue()->PostTask([this, &event] {
+ fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
+ event.Set();
+ });
+ ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
+ time_controller_->AdvanceTime(TimeDelta::Zero());
+ }
+
+ // Triggers resource usage measurements on the fake quality resource.
+ void TriggerQualityLow() {
+ rtc::Event event;
+ encoder_queue()->PostTask([this, &event] {
+ fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
+ event.Set();
+ });
+ ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
+ time_controller_->AdvanceTime(TimeDelta::Zero());
+ }
+ void TriggerQualityHigh() {
+ rtc::Event event;
+ encoder_queue()->PostTask([this, &event] {
+ fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
+ event.Set();
+ });
+ ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
+ time_controller_->AdvanceTime(TimeDelta::Zero());
+ }
+
+ TimeController* const time_controller_;
+ CpuOveruseDetectorProxy* overuse_detector_proxy_;
+ rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
+ rtc::scoped_refptr<FakeResource> fake_quality_resource_;
+ FakeAdaptationConstraint fake_adaptation_constraint_;
+};
+
+// Simulates simulcast behavior and makes highest stream resolutions divisible
+// by 4.
+class CroppingVideoStreamFactory
+ : public VideoEncoderConfig::VideoStreamFactoryInterface {
+ public:
+ CroppingVideoStreamFactory() {}
+
+ private:
+ std::vector<VideoStream> CreateEncoderStreams(
+ int frame_width,
+ int frame_height,
+ const VideoEncoderConfig& encoder_config) override {
+ std::vector<VideoStream> streams = test::CreateVideoStreams(
+ frame_width - frame_width % 4, frame_height - frame_height % 4,
+ encoder_config);
+ return streams;
+ }
+};
+
+class AdaptingFrameForwarder : public test::FrameForwarder {
+ public:
+ explicit AdaptingFrameForwarder(TimeController* time_controller)
+ : time_controller_(time_controller), adaptation_enabled_(false) {}
+ ~AdaptingFrameForwarder() override {}
+
+ void set_adaptation_enabled(bool enabled) {
+ MutexLock lock(&mutex_);
+ adaptation_enabled_ = enabled;
+ }
+
+ bool adaption_enabled() const {
+ MutexLock lock(&mutex_);
+ return adaptation_enabled_;
+ }
+
+ // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
+ // the resolution or frame rate was different than it is currently. If
+ // something else is modified, such as encoder resolutions, but the resolution
+ // and frame rate stays the same, last wants is not updated.
+ rtc::VideoSinkWants last_wants() const {
+ MutexLock lock(&mutex_);
+ return last_wants_;
+ }
+
+ absl::optional<int> last_sent_width() const { return last_width_; }
+ absl::optional<int> last_sent_height() const { return last_height_; }
+
+ void IncomingCapturedFrame(const VideoFrame& video_frame) override {
+ RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
+ time_controller_->AdvanceTime(TimeDelta::Zero());
+
+ int cropped_width = 0;
+ int cropped_height = 0;
+ int out_width = 0;
+ int out_height = 0;
+ if (adaption_enabled()) {
+ RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
+ << "w=" << video_frame.width()
+ << "h=" << video_frame.height();
+ if (adapter_.AdaptFrameResolution(
+ video_frame.width(), video_frame.height(),
+ video_frame.timestamp_us() * 1000, &cropped_width,
+ &cropped_height, &out_width, &out_height)) {
+ VideoFrame adapted_frame =
+ VideoFrame::Builder()
+ .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
+ nullptr, out_width, out_height))
+ .set_ntp_time_ms(video_frame.ntp_time_ms())
+ .set_timestamp_ms(99)
+ .set_rotation(kVideoRotation_0)
+ .build();
+ if (video_frame.has_update_rect()) {
+ adapted_frame.set_update_rect(
+ video_frame.update_rect().ScaleWithFrame(
+ video_frame.width(), video_frame.height(), 0, 0,
+ video_frame.width(), video_frame.height(), out_width,
+ out_height));
+ }
+ test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
+ last_width_.emplace(adapted_frame.width());
+ last_height_.emplace(adapted_frame.height());
+ } else {
+ last_width_ = absl::nullopt;
+ last_height_ = absl::nullopt;
+ }
+ } else {
+ RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
+ test::FrameForwarder::IncomingCapturedFrame(video_frame);
+ last_width_.emplace(video_frame.width());
+ last_height_.emplace(video_frame.height());
+ }
+ }
+
+ void OnOutputFormatRequest(int width, int height) {
+ absl::optional<std::pair<int, int>> target_aspect_ratio =
+ std::make_pair(width, height);
+ absl::optional<int> max_pixel_count = width * height;
+ absl::optional<int> max_fps;
+ adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
+ max_fps);
+ }
+
+ void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
+ const rtc::VideoSinkWants& wants) override {
+ MutexLock lock(&mutex_);
+ rtc::VideoSinkWants prev_wants = sink_wants_locked();
+ bool did_adapt =
+ prev_wants.max_pixel_count != wants.max_pixel_count ||
+ prev_wants.target_pixel_count != wants.target_pixel_count ||
+ prev_wants.max_framerate_fps != wants.max_framerate_fps;
+ if (did_adapt) {
+ last_wants_ = prev_wants;
+ }
+ adapter_.OnSinkWants(wants);
+ test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
+ }
+
+ void RequestRefreshFrame() override { ++refresh_frames_requested_; }
+
+ TimeController* const time_controller_;
+ cricket::VideoAdapter adapter_;
+ bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
+ rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
+ absl::optional<int> last_width_;
+ absl::optional<int> last_height_;
+ int refresh_frames_requested_{0};
+};
+
+// TODO(nisse): Mock only VideoStreamEncoderObserver.
+class MockableSendStatisticsProxy : public SendStatisticsProxy {
+ public:
+ MockableSendStatisticsProxy(Clock* clock,
+ const VideoSendStream::Config& config,
+ VideoEncoderConfig::ContentType content_type,
+ const FieldTrialsView& field_trials)
+ : SendStatisticsProxy(clock, config, content_type, field_trials) {}
+
+ VideoSendStream::Stats GetStats() override {
+ MutexLock lock(&lock_);
+ if (mock_stats_)
+ return *mock_stats_;
+ return SendStatisticsProxy::GetStats();
+ }
+
+ int GetInputFrameRate() const override {
+ MutexLock lock(&lock_);
+ if (mock_stats_)
+ return mock_stats_->input_frame_rate;
+ return SendStatisticsProxy::GetInputFrameRate();
+ }
+ void SetMockStats(const VideoSendStream::Stats& stats) {
+ MutexLock lock(&lock_);
+ mock_stats_.emplace(stats);
+ }
+
+ void ResetMockStats() {
+ MutexLock lock(&lock_);
+ mock_stats_.reset();
+ }
+
+ void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
+ on_frame_dropped_ = std::move(callback);
+ }
+
+ private:
+ void OnFrameDropped(DropReason reason) override {
+ SendStatisticsProxy::OnFrameDropped(reason);
+ if (on_frame_dropped_)
+ on_frame_dropped_(reason);
+ }
+
+ mutable Mutex lock_;
+ absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
+ std::function<void(DropReason)> on_frame_dropped_;
+};
+
+class SimpleVideoStreamEncoderFactory {
+ public:
+ class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
+ public:
+ using VideoStreamEncoder::VideoStreamEncoder;
+ ~AdaptedVideoStreamEncoder() { Stop(); }
+ };
+
+ class MockFakeEncoder : public test::FakeEncoder {
+ public:
+ using FakeEncoder::FakeEncoder;
+ MOCK_METHOD(CodecSpecificInfo,
+ EncodeHook,
+ (EncodedImage & encoded_image,
+ rtc::scoped_refptr<EncodedImageBuffer> buffer),
+ (override));
+ };
+
+ SimpleVideoStreamEncoderFactory() {
+ encoder_settings_.encoder_factory = &encoder_factory_;
+ encoder_settings_.bitrate_allocator_factory =
+ bitrate_allocator_factory_.get();
+ }
+
+ std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
+ std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
+ std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue,
+ const FieldTrialsView* field_trials = nullptr) {
+ auto result = std::make_unique<AdaptedVideoStreamEncoder>(
+ time_controller_.GetClock(),
+ /*number_of_cores=*/1,
+ /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
+ std::make_unique<CpuOveruseDetectorProxy>(
+ /*stats_proxy=*/nullptr,
+ field_trials ? *field_trials : field_trials_),
+ std::move(zero_hertz_adapter), std::move(encoder_queue),
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoBitrateAllocation,
+ field_trials ? *field_trials : field_trials_);
+ result->SetSink(&sink_, /*rotation_applied=*/false);
+ return result;
+ }
+
+ std::unique_ptr<AdaptedVideoStreamEncoder> Create(
+ std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
+ TaskQueueBase** encoder_queue_ptr = nullptr) {
+ auto encoder_queue =
+ time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
+ "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
+ if (encoder_queue_ptr)
+ *encoder_queue_ptr = encoder_queue.get();
+ return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
+ std::move(encoder_queue));
+ }
+
+ void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
+ MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
+
+ GlobalSimulatedTimeController* GetTimeController() {
+ return &time_controller_;
+ }
+
+ private:
+ class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
+ public:
+ ~NullEncoderSink() override = default;
+ void OnEncoderConfigurationChanged(
+ std::vector<VideoStream> streams,
+ bool is_svc,
+ VideoEncoderConfig::ContentType content_type,
+ int min_transmit_bitrate_bps) override {}
+ void OnBitrateAllocationUpdated(
+ const VideoBitrateAllocation& allocation) override {}
+ void OnVideoLayersAllocationUpdated(
+ VideoLayersAllocation allocation) override {}
+ Result OnEncodedImage(
+ const EncodedImage& encoded_image,
+ const CodecSpecificInfo* codec_specific_info) override {
+ return Result(EncodedImageCallback::Result::OK);
+ }
+ };
+
+ test::ScopedKeyValueConfig field_trials_;
+ GlobalSimulatedTimeController time_controller_{Timestamp::Zero()};
+ std::unique_ptr<TaskQueueFactory> task_queue_factory_{
+ time_controller_.CreateTaskQueueFactory()};
+ std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
+ std::make_unique<MockableSendStatisticsProxy>(
+ time_controller_.GetClock(),
+ VideoSendStream::Config(nullptr),
+ webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
+ field_trials_);
+ std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
+ CreateBuiltinVideoBitrateAllocatorFactory();
+ VideoStreamEncoderSettings encoder_settings_{
+ VideoEncoder::Capabilities(/*loss_notification=*/false)};
+ MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
+ test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
+ NullEncoderSink sink_;
+};
+
+class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
+ public:
+ MOCK_METHOD(void, Initialize, (Callback * callback), (override));
+ MOCK_METHOD(void,
+ SetZeroHertzModeEnabled,
+ (absl::optional<ZeroHertzModeParams>),
+ (override));
+ MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
+ MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
+ MOCK_METHOD(void, UpdateFrameRate, (), (override));
+ MOCK_METHOD(void,
+ UpdateLayerQualityConvergence,
+ (size_t spatial_index, bool converged),
+ (override));
+ MOCK_METHOD(void,
+ UpdateLayerStatus,
+ (size_t spatial_index, bool enabled),
+ (override));
+ MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
+};
+
+class MockEncoderSelector
+ : public VideoEncoderFactory::EncoderSelectorInterface {
+ public:
+ MOCK_METHOD(void,
+ OnCurrentEncoder,
+ (const SdpVideoFormat& format),
+ (override));
+ MOCK_METHOD(absl::optional<SdpVideoFormat>,
+ OnAvailableBitrate,
+ (const DataRate& rate),
+ (override));
+ MOCK_METHOD(absl::optional<SdpVideoFormat>,
+ OnResolutionChange,
+ (const RenderResolution& resolution),
+ (override));
+ MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
+};
+
+class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
+ public:
+ MOCK_METHOD(void,
+ AddOrUpdateSink,
+ (rtc::VideoSinkInterface<VideoFrame>*,
+ const rtc::VideoSinkWants&),
+ (override));
+ MOCK_METHOD(void,
+ RemoveSink,
+ (rtc::VideoSinkInterface<VideoFrame>*),
+ (override));
+ MOCK_METHOD(void, RequestRefreshFrame, (), (override));
+};
+
+} // namespace
+
+class VideoStreamEncoderTest : public ::testing::Test {
+ public:
+ static constexpr TimeDelta kDefaultTimeout = TimeDelta::Seconds(1);
+
+ VideoStreamEncoderTest()
+ : video_send_config_(VideoSendStream::Config(nullptr)),
+ codec_width_(320),
+ codec_height_(240),
+ max_framerate_(kDefaultFramerate),
+ fake_encoder_(&time_controller_),
+ encoder_factory_(&fake_encoder_),
+ stats_proxy_(new MockableSendStatisticsProxy(
+ time_controller_.GetClock(),
+ video_send_config_,
+ webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
+ field_trials_)),
+ sink_(&time_controller_, &fake_encoder_) {}
+
+ void SetUp() override {
+ metrics::Reset();
+ video_send_config_ = VideoSendStream::Config(nullptr);
+ video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
+ video_send_config_.encoder_settings.bitrate_allocator_factory =
+ &bitrate_allocator_factory_;
+ video_send_config_.rtp.payload_name = "FAKE";
+ video_send_config_.rtp.payload_type = 125;
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
+ EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
+ video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
+ video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
+ video_encoder_config_ = video_encoder_config.Copy();
+
+ ConfigureEncoder(std::move(video_encoder_config));
+ }
+
+ void ConfigureEncoder(
+ VideoEncoderConfig video_encoder_config,
+ VideoStreamEncoder::BitrateAllocationCallbackType
+ allocation_callback_type =
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoBitrateAllocationWhenScreenSharing,
+ int num_cores = 1) {
+ if (video_stream_encoder_)
+ video_stream_encoder_->Stop();
+
+ auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
+ "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
+ TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
+ std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
+ FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
+ encoder_queue_ptr, field_trials_);
+ video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
+ &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
+ stats_proxy_.get(), video_send_config_.encoder_settings,
+ allocation_callback_type, field_trials_, num_cores);
+ video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
+ video_stream_encoder_->SetSource(
+ &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
+ video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
+ kMaxPayloadLength, nullptr);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ }
+
+ void ResetEncoder(const std::string& payload_name,
+ size_t num_streams,
+ size_t num_temporal_layers,
+ unsigned char num_spatial_layers,
+ bool screenshare,
+ VideoStreamEncoder::BitrateAllocationCallbackType
+ allocation_callback_type =
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoBitrateAllocationWhenScreenSharing,
+ int num_cores = 1) {
+ video_send_config_.rtp.payload_name = payload_name;
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
+ num_streams, &video_encoder_config);
+ for (auto& layer : video_encoder_config.simulcast_layers) {
+ layer.num_temporal_layers = num_temporal_layers;
+ layer.max_framerate = kDefaultFramerate;
+ }
+ video_encoder_config.max_bitrate_bps =
+ num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
+ video_encoder_config.content_type =
+ screenshare ? VideoEncoderConfig::ContentType::kScreen
+ : VideoEncoderConfig::ContentType::kRealtimeVideo;
+ if (payload_name == "VP9") {
+ VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
+ vp9_settings.numberOfSpatialLayers = num_spatial_layers;
+ vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
+ video_encoder_config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
+ vp9_settings);
+ }
+ ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type,
+ num_cores);
+ }
+
+ VideoFrame CreateFrame(int64_t ntp_time_ms,
+ rtc::Event* destruction_event) const {
+ return VideoFrame::Builder()
+ .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
+ destruction_event, codec_width_, codec_height_))
+ .set_ntp_time_ms(ntp_time_ms)
+ .set_timestamp_ms(99)
+ .set_rotation(kVideoRotation_0)
+ .build();
+ }
+
+ VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
+ rtc::Event* destruction_event,
+ int offset_x) const {
+ return VideoFrame::Builder()
+ .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
+ destruction_event, codec_width_, codec_height_))
+ .set_ntp_time_ms(ntp_time_ms)
+ .set_timestamp_ms(99)
+ .set_rotation(kVideoRotation_0)
+ .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
+ .build();
+ }
+
+ VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
+ auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
+ I420Buffer::SetBlack(buffer.get());
+ return VideoFrame::Builder()
+ .set_video_frame_buffer(std::move(buffer))
+ .set_ntp_time_ms(ntp_time_ms)
+ .set_timestamp_ms(ntp_time_ms)
+ .set_rotation(kVideoRotation_0)
+ .build();
+ }
+
+ VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
+ return VideoFrame::Builder()
+ .set_video_frame_buffer(NV12Buffer::Create(width, height))
+ .set_ntp_time_ms(ntp_time_ms)
+ .set_timestamp_ms(ntp_time_ms)
+ .set_rotation(kVideoRotation_0)
+ .build();
+ }
+
+ VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
+ rtc::Event* destruction_event,
+ int width,
+ int height) const {
+ return VideoFrame::Builder()
+ .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
+ destruction_event, width, height))
+ .set_ntp_time_ms(ntp_time_ms)
+ .set_timestamp_ms(99)
+ .set_rotation(kVideoRotation_0)
+ .build();
+ }
+
+ VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
+ rtc::Event* destruction_event,
+ int width,
+ int height) const {
+ return VideoFrame::Builder()
+ .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
+ destruction_event, width, height))
+ .set_ntp_time_ms(ntp_time_ms)
+ .set_timestamp_ms(99)
+ .set_rotation(kVideoRotation_0)
+ .build();
+ }
+
+ VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
+ rtc::Event* destruction_event) const {
+ return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
+ codec_height_);
+ }
+
+ void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(1, codec_width_, codec_height_));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
+ }
+
+ void WaitForEncodedFrame(int64_t expected_ntp_time) {
+ sink_.WaitForEncodedFrame(expected_ntp_time);
+ AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
+ }
+
+ bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, TimeDelta timeout) {
+ bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout);
+ AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
+ return ok;
+ }
+
+ void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
+ sink_.WaitForEncodedFrame(expected_width, expected_height);
+ AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
+ }
+
+ void ExpectDroppedFrame() {
+ sink_.ExpectDroppedFrame();
+ AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
+ }
+
+ bool WaitForFrame(TimeDelta timeout) {
+ bool ok = sink_.WaitForFrame(timeout);
+ AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
+ return ok;
+ }
+
+ class TestEncoder : public test::FakeEncoder {
+ public:
+ explicit TestEncoder(TimeController* time_controller)
+ : FakeEncoder(time_controller->GetClock()),
+ time_controller_(time_controller) {
+ RTC_DCHECK(time_controller_);
+ }
+
+ VideoEncoder::EncoderInfo GetEncoderInfo() const override {
+ MutexLock lock(&local_mutex_);
+ EncoderInfo info = FakeEncoder::GetEncoderInfo();
+ if (initialized_ == EncoderState::kInitialized) {
+ if (quality_scaling_) {
+ info.scaling_settings = VideoEncoder::ScalingSettings(
+ kQpLow, kQpHigh, kMinPixelsPerFrame);
+ }
+ info.is_hardware_accelerated = is_hardware_accelerated_;
+ for (int i = 0; i < kMaxSpatialLayers; ++i) {
+ if (temporal_layers_supported_[i]) {
+ info.fps_allocation[i].clear();
+ int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
+ for (int tid = 0; tid < num_layers; ++tid)
+ info.fps_allocation[i].push_back(255 / (num_layers - tid));
+ }
+ }
+ }
+
+ info.resolution_bitrate_limits = resolution_bitrate_limits_;
+ info.requested_resolution_alignment = requested_resolution_alignment_;
+ info.apply_alignment_to_all_simulcast_layers =
+ apply_alignment_to_all_simulcast_layers_;
+ info.preferred_pixel_formats = preferred_pixel_formats_;
+ if (is_qp_trusted_.has_value()) {
+ info.is_qp_trusted = is_qp_trusted_;
+ }
+ return info;
+ }
+
+ int32_t RegisterEncodeCompleteCallback(
+ EncodedImageCallback* callback) override {
+ MutexLock lock(&local_mutex_);
+ encoded_image_callback_ = callback;
+ return FakeEncoder::RegisterEncodeCompleteCallback(callback);
+ }
+
+ void ContinueEncode() { continue_encode_event_.Set(); }
+
+ void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
+ uint32_t timestamp) const {
+ MutexLock lock(&local_mutex_);
+ EXPECT_EQ(timestamp_, timestamp);
+ EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
+ }
+
+ void SetQualityScaling(bool b) {
+ MutexLock lock(&local_mutex_);
+ quality_scaling_ = b;
+ }
+
+ void SetRequestedResolutionAlignment(
+ uint32_t requested_resolution_alignment) {
+ MutexLock lock(&local_mutex_);
+ requested_resolution_alignment_ = requested_resolution_alignment;
+ }
+
+ void SetApplyAlignmentToAllSimulcastLayers(bool b) {
+ MutexLock lock(&local_mutex_);
+ apply_alignment_to_all_simulcast_layers_ = b;
+ }
+
+ void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
+ MutexLock lock(&local_mutex_);
+ is_hardware_accelerated_ = is_hardware_accelerated;
+ }
+
+ void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
+ RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
+ MutexLock lock(&local_mutex_);
+ temporal_layers_supported_[spatial_idx] = supported;
+ }
+
+ void SetResolutionBitrateLimits(
+ std::vector<ResolutionBitrateLimits> thresholds) {
+ MutexLock lock(&local_mutex_);
+ resolution_bitrate_limits_ = thresholds;
+ }
+
+ void ForceInitEncodeFailure(bool force_failure) {
+ MutexLock lock(&local_mutex_);
+ force_init_encode_failed_ = force_failure;
+ }
+
+ void SimulateOvershoot(double rate_factor) {
+ MutexLock lock(&local_mutex_);
+ rate_factor_ = rate_factor;
+ }
+
+ uint32_t GetLastFramerate() const {
+ MutexLock lock(&local_mutex_);
+ return last_framerate_;
+ }
+
+ VideoFrame::UpdateRect GetLastUpdateRect() const {
+ MutexLock lock(&local_mutex_);
+ return last_update_rect_;
+ }
+
+ const std::vector<VideoFrameType>& LastFrameTypes() const {
+ MutexLock lock(&local_mutex_);
+ return last_frame_types_;
+ }
+
+ void InjectFrame(const VideoFrame& input_image, bool keyframe) {
+ const std::vector<VideoFrameType> frame_type = {
+ keyframe ? VideoFrameType::kVideoFrameKey
+ : VideoFrameType::kVideoFrameDelta};
+ {
+ MutexLock lock(&local_mutex_);
+ last_frame_types_ = frame_type;
+ }
+ FakeEncoder::Encode(input_image, &frame_type);
+ }
+
+ void InjectEncodedImage(const EncodedImage& image,
+ const CodecSpecificInfo* codec_specific_info) {
+ MutexLock lock(&local_mutex_);
+ encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
+ }
+
+ void SetEncodedImageData(
+ rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
+ MutexLock lock(&local_mutex_);
+ encoded_image_data_ = encoded_image_data;
+ }
+
+ void ExpectNullFrame() {
+ MutexLock lock(&local_mutex_);
+ expect_null_frame_ = true;
+ }
+
+ absl::optional<VideoEncoder::RateControlParameters>
+ GetAndResetLastRateControlSettings() {
+ auto settings = last_rate_control_settings_;
+ last_rate_control_settings_.reset();
+ return settings;
+ }
+
+ int GetLastInputWidth() const {
+ MutexLock lock(&local_mutex_);
+ return last_input_width_;
+ }
+
+ int GetLastInputHeight() const {
+ MutexLock lock(&local_mutex_);
+ return last_input_height_;
+ }
+
+ absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
+ MutexLock lock(&local_mutex_);
+ return last_input_pixel_format_;
+ }
+
+ int GetNumSetRates() const {
+ MutexLock lock(&local_mutex_);
+ return num_set_rates_;
+ }
+
+ void SetPreferredPixelFormats(
+ absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
+ pixel_formats) {
+ MutexLock lock(&local_mutex_);
+ preferred_pixel_formats_ = std::move(pixel_formats);
+ }
+
+ void SetIsQpTrusted(absl::optional<bool> trusted) {
+ MutexLock lock(&local_mutex_);
+ is_qp_trusted_ = trusted;
+ }
+
+ VideoCodecComplexity LastEncoderComplexity() {
+ MutexLock lock(&local_mutex_);
+ return last_encoder_complexity_;
+ }
+
+ private:
+ int32_t Encode(const VideoFrame& input_image,
+ const std::vector<VideoFrameType>* frame_types) override {
+ {
+ MutexLock lock(&local_mutex_);
+ if (expect_null_frame_) {
+ EXPECT_EQ(input_image.timestamp(), 0u);
+ EXPECT_EQ(input_image.width(), 1);
+ last_frame_types_ = *frame_types;
+ expect_null_frame_ = false;
+ } else {
+ EXPECT_GT(input_image.timestamp(), timestamp_);
+ EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
+ EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
+ }
+
+ timestamp_ = input_image.timestamp();
+ ntp_time_ms_ = input_image.ntp_time_ms();
+ last_input_width_ = input_image.width();
+ last_input_height_ = input_image.height();
+ last_update_rect_ = input_image.update_rect();
+ last_frame_types_ = *frame_types;
+ last_input_pixel_format_ = input_image.video_frame_buffer()->type();
+ }
+ int32_t result = FakeEncoder::Encode(input_image, frame_types);
+ return result;
+ }
+
+ CodecSpecificInfo EncodeHook(
+ EncodedImage& encoded_image,
+ rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
+ CodecSpecificInfo codec_specific;
+ {
+ MutexLock lock(&mutex_);
+ codec_specific.codecType = config_.codecType;
+ }
+ MutexLock lock(&local_mutex_);
+ if (encoded_image_data_) {
+ encoded_image.SetEncodedData(encoded_image_data_);
+ }
+ return codec_specific;
+ }
+
+ int32_t InitEncode(const VideoCodec* config,
+ const Settings& settings) override {
+ int res = FakeEncoder::InitEncode(config, settings);
+
+ MutexLock lock(&local_mutex_);
+ EXPECT_EQ(initialized_, EncoderState::kUninitialized);
+
+ if (config->codecType == kVideoCodecVP8) {
+ // Simulate setting up temporal layers, in order to validate the life
+ // cycle of these objects.
+ Vp8TemporalLayersFactory factory;
+ frame_buffer_controller_ =
+ factory.Create(*config, settings, &fec_controller_override_);
+ }
+
+ last_encoder_complexity_ = config->GetVideoEncoderComplexity();
+
+ if (force_init_encode_failed_) {
+ initialized_ = EncoderState::kInitializationFailed;
+ return -1;
+ }
+
+ initialized_ = EncoderState::kInitialized;
+ return res;
+ }
+
+ int32_t Release() override {
+ MutexLock lock(&local_mutex_);
+ EXPECT_NE(initialized_, EncoderState::kUninitialized);
+ initialized_ = EncoderState::kUninitialized;
+ return FakeEncoder::Release();
+ }
+
+ void SetRates(const RateControlParameters& parameters) {
+ MutexLock lock(&local_mutex_);
+ num_set_rates_++;
+ VideoBitrateAllocation adjusted_rate_allocation;
+ for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
+ for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
+ if (parameters.bitrate.HasBitrate(si, ti)) {
+ adjusted_rate_allocation.SetBitrate(
+ si, ti,
+ static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
+ rate_factor_));
+ }
+ }
+ }
+ last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
+ last_rate_control_settings_ = parameters;
+ RateControlParameters adjusted_paramters = parameters;
+ adjusted_paramters.bitrate = adjusted_rate_allocation;
+ FakeEncoder::SetRates(adjusted_paramters);
+ }
+
+ TimeController* const time_controller_;
+ mutable Mutex local_mutex_;
+ enum class EncoderState {
+ kUninitialized,
+ kInitializationFailed,
+ kInitialized
+ } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
+ rtc::Event continue_encode_event_;
+ uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
+ int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
+ int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
+ int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
+ bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
+ uint32_t requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
+ bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
+ false;
+ bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
+ rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
+ RTC_GUARDED_BY(local_mutex_);
+ std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
+ RTC_GUARDED_BY(local_mutex_);
+ absl::optional<bool>
+ temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
+ local_mutex_);
+ bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
+ double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
+ uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
+ absl::optional<VideoEncoder::RateControlParameters>
+ last_rate_control_settings_;
+ VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
+ 0, 0, 0, 0};
+ std::vector<VideoFrameType> last_frame_types_;
+ bool expect_null_frame_ = false;
+ EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
+ nullptr;
+ NiceMock<MockFecControllerOverride> fec_controller_override_;
+ std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
+ RTC_GUARDED_BY(local_mutex_);
+ int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
+ absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
+ RTC_GUARDED_BY(local_mutex_);
+ absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
+ preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
+ absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
+ VideoCodecComplexity last_encoder_complexity_ RTC_GUARDED_BY(local_mutex_){
+ VideoCodecComplexity::kComplexityNormal};
+ };
+
+ class TestSink : public VideoStreamEncoder::EncoderSink {
+ public:
+ TestSink(TimeController* time_controller, TestEncoder* test_encoder)
+ : time_controller_(time_controller), test_encoder_(test_encoder) {
+ RTC_DCHECK(time_controller_);
+ }
+
+ void WaitForEncodedFrame(int64_t expected_ntp_time) {
+ EXPECT_TRUE(TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeout));
+ }
+
+ bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
+ TimeDelta timeout) {
+ uint32_t timestamp = 0;
+ if (!WaitForFrame(timeout))
+ return false;
+ {
+ MutexLock lock(&mutex_);
+ timestamp = last_timestamp_;
+ }
+ test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
+ return true;
+ }
+
+ void WaitForEncodedFrame(uint32_t expected_width,
+ uint32_t expected_height) {
+ EXPECT_TRUE(WaitForFrame(kDefaultTimeout));
+ CheckLastFrameSizeMatches(expected_width, expected_height);
+ }
+
+ void CheckLastFrameSizeMatches(uint32_t expected_width,
+ uint32_t expected_height) {
+ uint32_t width = 0;
+ uint32_t height = 0;
+ {
+ MutexLock lock(&mutex_);
+ width = last_width_;
+ height = last_height_;
+ }
+ EXPECT_EQ(expected_height, height);
+ EXPECT_EQ(expected_width, width);
+ }
+
+ void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
+ VideoRotation rotation;
+ {
+ MutexLock lock(&mutex_);
+ rotation = last_rotation_;
+ }
+ EXPECT_EQ(expected_rotation, rotation);
+ }
+
+ void ExpectDroppedFrame() {
+ EXPECT_FALSE(WaitForFrame(TimeDelta::Millis(100)));
+ }
+
+ bool WaitForFrame(TimeDelta timeout) {
+ RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
+ time_controller_->AdvanceTime(TimeDelta::Zero());
+ bool ret = encoded_frame_event_.Wait(timeout);
+ time_controller_->AdvanceTime(TimeDelta::Zero());
+ return ret;
+ }
+
+ void SetExpectNoFrames() {
+ MutexLock lock(&mutex_);
+ expect_frames_ = false;
+ }
+
+ int number_of_reconfigurations() const {
+ MutexLock lock(&mutex_);
+ return number_of_reconfigurations_;
+ }
+
+ int last_min_transmit_bitrate() const {
+ MutexLock lock(&mutex_);
+ return min_transmit_bitrate_bps_;
+ }
+
+ void SetNumExpectedLayers(size_t num_layers) {
+ MutexLock lock(&mutex_);
+ num_expected_layers_ = num_layers;
+ }
+
+ int64_t GetLastCaptureTimeMs() const {
+ MutexLock lock(&mutex_);
+ return last_capture_time_ms_;
+ }
+
+ const EncodedImage& GetLastEncodedImage() {
+ MutexLock lock(&mutex_);
+ return last_encoded_image_;
+ }
+
+ std::vector<uint8_t> GetLastEncodedImageData() {
+ MutexLock lock(&mutex_);
+ return std::move(last_encoded_image_data_);
+ }
+
+ VideoBitrateAllocation GetLastVideoBitrateAllocation() {
+ MutexLock lock(&mutex_);
+ return last_bitrate_allocation_;
+ }
+
+ int number_of_bitrate_allocations() const {
+ MutexLock lock(&mutex_);
+ return number_of_bitrate_allocations_;
+ }
+
+ VideoLayersAllocation GetLastVideoLayersAllocation() {
+ MutexLock lock(&mutex_);
+ return last_layers_allocation_;
+ }
+
+ int number_of_layers_allocations() const {
+ MutexLock lock(&mutex_);
+ return number_of_layers_allocations_;
+ }
+
+ private:
+ Result OnEncodedImage(
+ const EncodedImage& encoded_image,
+ const CodecSpecificInfo* codec_specific_info) override {
+ MutexLock lock(&mutex_);
+ EXPECT_TRUE(expect_frames_);
+ last_encoded_image_ = EncodedImage(encoded_image);
+ last_encoded_image_data_ = std::vector<uint8_t>(
+ encoded_image.data(), encoded_image.data() + encoded_image.size());
+ uint32_t timestamp = encoded_image.Timestamp();
+ if (last_timestamp_ != timestamp) {
+ num_received_layers_ = 1;
+ last_width_ = encoded_image._encodedWidth;
+ last_height_ = encoded_image._encodedHeight;
+ } else {
+ ++num_received_layers_;
+ last_width_ = std::max(encoded_image._encodedWidth, last_width_);
+ last_height_ = std::max(encoded_image._encodedHeight, last_height_);
+ }
+ last_timestamp_ = timestamp;
+ last_capture_time_ms_ = encoded_image.capture_time_ms_;
+ last_rotation_ = encoded_image.rotation_;
+ if (num_received_layers_ == num_expected_layers_) {
+ encoded_frame_event_.Set();
+ }
+ return Result(Result::OK, last_timestamp_);
+ }
+
+ void OnEncoderConfigurationChanged(
+ std::vector<VideoStream> streams,
+ bool is_svc,
+ VideoEncoderConfig::ContentType content_type,
+ int min_transmit_bitrate_bps) override {
+ MutexLock lock(&mutex_);
+ ++number_of_reconfigurations_;
+ min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
+ }
+
+ void OnBitrateAllocationUpdated(
+ const VideoBitrateAllocation& allocation) override {
+ MutexLock lock(&mutex_);
+ ++number_of_bitrate_allocations_;
+ last_bitrate_allocation_ = allocation;
+ }
+
+ void OnVideoLayersAllocationUpdated(
+ VideoLayersAllocation allocation) override {
+ MutexLock lock(&mutex_);
+ ++number_of_layers_allocations_;
+ last_layers_allocation_ = allocation;
+ rtc::StringBuilder log;
+ for (const auto& layer : allocation.active_spatial_layers) {
+ log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
+ << "[";
+ for (const auto target_bitrate :
+ layer.target_bitrate_per_temporal_layer) {
+ log << target_bitrate.kbps() << ",";
+ }
+ log << "]";
+ }
+ RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
+ }
+
+ TimeController* const time_controller_;
+ mutable Mutex mutex_;
+ TestEncoder* test_encoder_;
+ rtc::Event encoded_frame_event_;
+ EncodedImage last_encoded_image_;
+ std::vector<uint8_t> last_encoded_image_data_;
+ uint32_t last_timestamp_ = 0;
+ int64_t last_capture_time_ms_ = 0;
+ uint32_t last_height_ = 0;
+ uint32_t last_width_ = 0;
+ VideoRotation last_rotation_ = kVideoRotation_0;
+ size_t num_expected_layers_ = 1;
+ size_t num_received_layers_ = 0;
+ bool expect_frames_ = true;
+ int number_of_reconfigurations_ = 0;
+ int min_transmit_bitrate_bps_ = 0;
+ VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
+ int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
+ VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
+ int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
+ };
+
+ class VideoBitrateAllocatorProxyFactory
+ : public VideoBitrateAllocatorFactory {
+ public:
+ VideoBitrateAllocatorProxyFactory()
+ : bitrate_allocator_factory_(
+ CreateBuiltinVideoBitrateAllocatorFactory()) {}
+
+ std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
+ const VideoCodec& codec) override {
+ MutexLock lock(&mutex_);
+ codec_config_ = codec;
+ return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
+ }
+
+ VideoCodec codec_config() const {
+ MutexLock lock(&mutex_);
+ return codec_config_;
+ }
+
+ private:
+ std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
+
+ mutable Mutex mutex_;
+ VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
+ };
+
+ Clock* clock() { return time_controller_.GetClock(); }
+ void AdvanceTime(TimeDelta duration) {
+ time_controller_.AdvanceTime(duration);
+ }
+
+ int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
+
+ protected:
+ virtual TaskQueueFactory* GetTaskQueueFactory() {
+ return time_controller_.GetTaskQueueFactory();
+ }
+
+ test::ScopedKeyValueConfig field_trials_;
+ GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
+ VideoSendStream::Config video_send_config_;
+ VideoEncoderConfig video_encoder_config_;
+ int codec_width_;
+ int codec_height_;
+ int max_framerate_;
+ TestEncoder fake_encoder_;
+ test::VideoEncoderProxyFactory encoder_factory_;
+ VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
+ std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
+ TestSink sink_;
+ AdaptingFrameForwarder video_source_{&time_controller_};
+ std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
+};
+
+TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ rtc::Event frame_destroyed_event;
+ video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
+ WaitForEncodedFrame(1);
+ EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
+ // Dropped since no target bitrate has been set.
+ rtc::Event frame_destroyed_event;
+ // The encoder will cache up to one frame for a short duration. Adding two
+ // frames means that the first frame will be dropped and the second frame will
+ // be sent when the encoder is enabled.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
+ AdvanceTime(TimeDelta::Millis(10));
+ video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
+ AdvanceTime(TimeDelta::Zero());
+ EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // The pending frame should be received.
+ WaitForEncodedFrame(2);
+ video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
+
+ WaitForEncodedFrame(3);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
+
+ // The encoder will cache up to one frame for a short duration. Adding two
+ // frames means that the first frame will be dropped and the second frame will
+ // be sent when the encoder is resumed.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
+ video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ WaitForEncodedFrame(3);
+ video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
+ WaitForEncodedFrame(4);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+
+ // This frame will be dropped since it has the same ntp timestamp.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+
+ video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
+ WaitForEncodedFrame(2);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+
+ video_stream_encoder_->Stop();
+ sink_.SetExpectNoFrames();
+ rtc::Event frame_destroyed_event;
+ video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
+ EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
+}
+
+TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(&source,
+ DegradationPreference::MAINTAIN_FRAMERATE);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ int dropped_count = 0;
+ stats_proxy_->SetDroppedFrameCallback(
+ [&dropped_count](VideoStreamEncoderObserver::DropReason) {
+ ++dropped_count;
+ });
+
+ source.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ source.IncomingCapturedFrame(CreateFrame(2, nullptr));
+ WaitForEncodedFrame(2);
+ video_stream_encoder_->Stop();
+ EXPECT_EQ(1, dropped_count);
+}
+
+TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ rtc::Event frame_destroyed_event;
+ video_source_.IncomingCapturedFrame(
+ CreateFakeNativeFrame(1, &frame_destroyed_event));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(VideoFrameBuffer::Type::kNative,
+ fake_encoder_.GetLastInputPixelFormat());
+ EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
+ EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
+ // Use the cropping factory.
+ video_encoder_config_.video_stream_factory =
+ rtc::make_ref_counted<CroppingVideoStreamFactory>();
+ video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
+ kMaxPayloadLength);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ // Capture a frame at codec_width_/codec_height_.
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ // The encoder will have been configured once.
+ EXPECT_EQ(1, sink_.number_of_reconfigurations());
+ EXPECT_EQ(codec_width_, fake_encoder_.config().width);
+ EXPECT_EQ(codec_height_, fake_encoder_.config().height);
+
+ // Now send in a fake frame that needs to be cropped as the width/height
+ // aren't divisible by 4 (see CreateEncoderStreams above).
+ rtc::Event frame_destroyed_event;
+ video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
+ 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
+ WaitForEncodedFrame(2);
+ EXPECT_EQ(VideoFrameBuffer::Type::kNative,
+ fake_encoder_.GetLastInputPixelFormat());
+ EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
+ EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(
+ CreateNV12Frame(1, codec_width_, codec_height_));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
+ fake_encoder_.GetLastInputPixelFormat());
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ fake_encoder_.SetPreferredPixelFormats({});
+
+ rtc::Event frame_destroyed_event;
+ video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
+ 1, &frame_destroyed_event, codec_width_, codec_height_));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(VideoFrameBuffer::Type::kNative,
+ fake_encoder_.GetLastInputPixelFormat());
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
+
+ rtc::Event frame_destroyed_event;
+ video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
+ 1, &frame_destroyed_event, codec_width_, codec_height_));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(VideoFrameBuffer::Type::kNative,
+ fake_encoder_.GetLastInputPixelFormat());
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Fake NV12 native frame does not allow mapping to I444.
+ fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
+
+ rtc::Event frame_destroyed_event;
+ video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
+ 1, &frame_destroyed_event, codec_width_, codec_height_));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(VideoFrameBuffer::Type::kNative,
+ fake_encoder_.GetLastInputPixelFormat());
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ rtc::Event frame_destroyed_event;
+ video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
+ 1, &frame_destroyed_event, codec_width_, codec_height_));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(VideoFrameBuffer::Type::kNative,
+ fake_encoder_.GetLastInputPixelFormat());
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
+ // The congestion window pushback is set to 0.5, which will drop 1/2 of
+ // frames. Adding two frames means that the first frame will be dropped and
+ // the second frame will be sent to the encoder.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
+ video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
+ WaitForEncodedFrame(3);
+ video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
+ video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
+ WaitForEncodedFrame(5);
+ EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ EXPECT_EQ(0, sink_.number_of_reconfigurations());
+
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ // The encoder will have been configured once when the first frame is
+ // received.
+ EXPECT_EQ(1, sink_.number_of_reconfigurations());
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
+ video_encoder_config.min_transmit_bitrate_bps = 9999;
+ video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
+ kMaxPayloadLength);
+
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
+ WaitForEncodedFrame(2);
+ EXPECT_EQ(2, sink_.number_of_reconfigurations());
+ EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ // The encoder will have been configured once.
+ EXPECT_EQ(1, sink_.number_of_reconfigurations());
+ EXPECT_EQ(codec_width_, fake_encoder_.config().width);
+ EXPECT_EQ(codec_height_, fake_encoder_.config().height);
+
+ codec_width_ *= 2;
+ codec_height_ *= 2;
+ // Capture a frame with a higher resolution and wait for it to synchronize
+ // with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
+ WaitForEncodedFrame(2);
+ EXPECT_EQ(codec_width_, fake_encoder_.config().width);
+ EXPECT_EQ(codec_height_, fake_encoder_.config().height);
+ EXPECT_EQ(2, sink_.number_of_reconfigurations());
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
+ // Changing the max payload data length recreates encoder.
+ video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
+ kMaxPayloadLength / 2);
+
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
+ WaitForEncodedFrame(2);
+ EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
+ video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ // The encoder will have been configured once when the first frame is
+ // received.
+ EXPECT_EQ(1, sink_.number_of_reconfigurations());
+ EXPECT_EQ(kTargetBitrate.bps(),
+ bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
+ EXPECT_EQ(kStartBitrate.bps(),
+ bitrate_allocator_factory_.codec_config().startBitrate * 1000);
+
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1,
+ &video_encoder_config); //???
+ video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
+ video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
+ video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
+ kMaxPayloadLength);
+
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
+ WaitForEncodedFrame(2);
+ EXPECT_EQ(2, sink_.number_of_reconfigurations());
+ // Bitrate limits have changed - rate allocator should be reconfigured,
+ // encoder should not be reconfigured.
+ EXPECT_EQ(kTargetBitrate.bps() * 2,
+ bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
+ EXPECT_EQ(kStartBitrate.bps() * 2,
+ bitrate_allocator_factory_.codec_config().startBitrate * 1000);
+ EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ const uint32_t kMinEncBitrateKbps = 100;
+ const uint32_t kMaxEncBitrateKbps = 1000;
+ const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
+ /*frame_size_pixels=*/codec_width_ * codec_height_,
+ /*min_start_bitrate_bps=*/0,
+ /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
+ /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
+ fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
+ video_encoder_config.simulcast_layers[0].min_bitrate_bps =
+ (kMinEncBitrateKbps + 1) * 1000;
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+
+ // When both encoder and app provide bitrate limits, the intersection of
+ // provided sets should be used.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(kMaxEncBitrateKbps,
+ bitrate_allocator_factory_.codec_config().maxBitrate);
+ EXPECT_EQ(kMinEncBitrateKbps + 1,
+ bitrate_allocator_factory_.codec_config().minBitrate);
+
+ video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
+ video_encoder_config.simulcast_layers[0].min_bitrate_bps =
+ (kMinEncBitrateKbps - 1) * 1000;
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+ video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
+ WaitForEncodedFrame(2);
+ EXPECT_EQ(kMaxEncBitrateKbps - 1,
+ bitrate_allocator_factory_.codec_config().maxBitrate);
+ EXPECT_EQ(kMinEncBitrateKbps,
+ bitrate_allocator_factory_.codec_config().minBitrate);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ const uint32_t kMinAppBitrateKbps = 100;
+ const uint32_t kMaxAppBitrateKbps = 200;
+ const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
+ const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
+ const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
+ /*frame_size_pixels=*/codec_width_ * codec_height_,
+ /*min_start_bitrate_bps=*/0,
+ /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
+ /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
+ fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
+ video_encoder_config.simulcast_layers[0].min_bitrate_bps =
+ kMinAppBitrateKbps * 1000;
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(kMaxAppBitrateKbps,
+ bitrate_allocator_factory_.codec_config().maxBitrate);
+ EXPECT_EQ(kMinAppBitrateKbps,
+ bitrate_allocator_factory_.codec_config().minBitrate);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
+ 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
+ const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
+ 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
+ fake_encoder_.SetResolutionBitrateLimits(
+ {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = 0;
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+
+ // 270p. The bitrate limits recommended by encoder for 270p should be used.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
+ bitrate_allocator_factory_.codec_config().minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
+ bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
+
+ // 360p. The bitrate limits recommended by encoder for 360p should be used.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
+ WaitForEncodedFrame(2);
+ EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
+ bitrate_allocator_factory_.codec_config().minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
+ bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
+
+ // Resolution between 270p and 360p. The bitrate limits recommended by
+ // encoder for 360p should be used.
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
+ WaitForEncodedFrame(3);
+ EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
+ bitrate_allocator_factory_.codec_config().minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
+ bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
+
+ // Resolution higher than 360p. The caps recommended by encoder should be
+ // ignored.
+ video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
+ WaitForEncodedFrame(4);
+ EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
+ bitrate_allocator_factory_.codec_config().minBitrate * 1000);
+ EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
+ bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
+ EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
+ bitrate_allocator_factory_.codec_config().minBitrate * 1000);
+ EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
+ bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
+
+ // Resolution lower than 270p. The max bitrate limit recommended by encoder
+ // for 270p should be used.
+ video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
+ WaitForEncodedFrame(5);
+ EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
+ bitrate_allocator_factory_.codec_config().minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
+ bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = 0;
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+
+ // Encode 720p frame to get the default encoder target bitrate.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ const uint32_t kDefaultTargetBitrateFor720pKbps =
+ bitrate_allocator_factory_.codec_config()
+ .simulcastStream[0]
+ .targetBitrate;
+
+ // Set the max recommended encoder bitrate to something lower than the default
+ // target bitrate.
+ const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
+ 1280 * 720, 10 * 1000, 10 * 1000,
+ kDefaultTargetBitrateFor720pKbps / 2 * 1000);
+ fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
+
+ // Change resolution to trigger encoder reinitialization.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
+ WaitForEncodedFrame(2);
+ video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
+ WaitForEncodedFrame(3);
+
+ // Ensure the target bitrate is capped by the max bitrate.
+ EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
+ static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
+ EXPECT_EQ(bitrate_allocator_factory_.codec_config()
+ .simulcastStream[0]
+ .targetBitrate *
+ 1000,
+ static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
+ const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
+ 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
+ const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
+ 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
+ fake_encoder_.SetResolutionBitrateLimits(
+ {kEncoderLimits270p, kEncoderLimits360p});
+
+ // Two streams, highest stream active.
+ VideoEncoderConfig config;
+ webrtc::VideoEncoder::EncoderInfo encoder_info;
+ const int kNumStreams = 2;
+ test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
+ config.max_bitrate_bps = 0;
+ config.simulcast_layers[0].active = false;
+ config.simulcast_layers[1].active = true;
+ config.video_stream_factory =
+ rtc::make_ref_counted<cricket::EncoderStreamFactory>(
+ "VP8", /*max qp*/ 56, /*screencast*/ false,
+ /*screenshare enabled*/ false, encoder_info);
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+
+ // The encoder bitrate limits for 270p should be used.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
+
+ // The encoder bitrate limits for 360p should be used.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
+
+ // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
+
+ // Resolution higher than 360p. Encoder limits should be ignored.
+ video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
+ EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
+ EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
+ EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
+
+ // Resolution lower than 270p. The encoder limits for 270p should be used.
+ video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
+ // Two streams, highest stream active.
+ VideoEncoderConfig config;
+ webrtc::VideoEncoder::EncoderInfo encoder_info;
+ const int kNumStreams = 2;
+ test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
+ config.max_bitrate_bps = 0;
+ config.simulcast_layers[0].active = false;
+ config.simulcast_layers[1].active = true;
+ config.video_stream_factory =
+ rtc::make_ref_counted<cricket::EncoderStreamFactory>(
+ "VP8", /*max qp*/ 56, /*screencast*/ false,
+ /*screenshare enabled*/ false, encoder_info);
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+
+ // Default bitrate limits for 270p should be used.
+ const absl::optional<VideoEncoder::ResolutionBitrateLimits>
+ kDefaultLimits270p =
+ EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
+ kVideoCodecVP8, 480 * 270);
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
+ EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
+
+ // Default bitrate limits for 360p should be used.
+ const absl::optional<VideoEncoder::ResolutionBitrateLimits>
+ kDefaultLimits360p =
+ EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
+ kVideoCodecVP8, 640 * 360);
+ video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
+
+ // Resolution b/w 270p and 360p. The default limits for 360p should be used.
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
+
+ // Default bitrate limits for 540p should be used.
+ const absl::optional<VideoEncoder::ResolutionBitrateLimits>
+ kDefaultLimits540p =
+ EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
+ kVideoCodecVP8, 960 * 540);
+ video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
+ const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
+ 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
+ const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
+ 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
+ const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
+ 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
+ fake_encoder_.SetResolutionBitrateLimits(
+ {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
+
+ // Three streams, middle stream active.
+ VideoEncoderConfig config;
+ webrtc::VideoEncoder::EncoderInfo encoder_info;
+ const int kNumStreams = 3;
+ test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
+ config.simulcast_layers[0].active = false;
+ config.simulcast_layers[1].active = true;
+ config.simulcast_layers[2].active = false;
+ config.video_stream_factory =
+ rtc::make_ref_counted<cricket::EncoderStreamFactory>(
+ "VP8", /*max qp*/ 56, /*screencast*/ false,
+ /*screenshare enabled*/ false, encoder_info);
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+
+ // The encoder bitrate limits for 360p should be used.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
+
+ // The encoder bitrate limits for 270p should be used.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
+ const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
+ 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
+ const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
+ 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
+ const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
+ 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
+ fake_encoder_.SetResolutionBitrateLimits(
+ {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
+
+ // Three streams, lowest stream active.
+ VideoEncoderConfig config;
+ webrtc::VideoEncoder::EncoderInfo encoder_info;
+ const int kNumStreams = 3;
+ test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
+ config.simulcast_layers[0].active = true;
+ config.simulcast_layers[1].active = false;
+ config.simulcast_layers[2].active = false;
+ config.video_stream_factory =
+ rtc::make_ref_counted<cricket::EncoderStreamFactory>(
+ "VP8", /*max qp*/ 56, /*screencast*/ false,
+ /*screenshare enabled*/ false, encoder_info);
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+
+ // Resolution on lowest stream lower than 270p. The encoder limits not applied
+ // on lowest stream, limits for 270p should not be used
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
+ EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
+ EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
+ const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
+ 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
+ const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
+ 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
+ fake_encoder_.SetResolutionBitrateLimits(
+ {kEncoderLimits270p, kEncoderLimits360p});
+ const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
+
+ // Two streams, highest stream active.
+ VideoEncoderConfig config;
+ webrtc::VideoEncoder::EncoderInfo encoder_info;
+ const int kNumStreams = 2;
+ test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
+ config.simulcast_layers[0].active = false;
+ config.simulcast_layers[1].active = true;
+ config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
+ config.video_stream_factory =
+ rtc::make_ref_counted<cricket::EncoderStreamFactory>(
+ "VP8", /*max qp*/ 56, /*screencast*/ false,
+ /*screenshare enabled*/ false, encoder_info);
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+
+ // The encoder bitrate limits for 270p should be used.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
+
+ // The max configured bitrate is less than the encoder limit for 360p.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
+ fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
+ fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
+ EXPECT_TRUE(video_source_.has_sinks());
+ test::FrameForwarder new_video_source;
+ video_stream_encoder_->SetSource(
+ &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_FALSE(video_source_.has_sinks());
+ EXPECT_TRUE(new_video_source.has_sinks());
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
+ EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
+ video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
+ EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
+ video_stream_encoder_->Stop();
+}
+
+class ResolutionAlignmentTest
+ : public VideoStreamEncoderTest,
+ public ::testing::WithParamInterface<
+ ::testing::tuple<int, std::vector<double>>> {
+ public:
+ ResolutionAlignmentTest()
+ : requested_alignment_(::testing::get<0>(GetParam())),
+ scale_factors_(::testing::get<1>(GetParam())) {}
+
+ protected:
+ const uint32_t requested_alignment_;
+ const std::vector<double> scale_factors_;
+};
+
+INSTANTIATE_TEST_SUITE_P(
+ AlignmentAndScaleFactors,
+ ResolutionAlignmentTest,
+ ::testing::Combine(
+ ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
+ ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
+ std::vector<double>{-1.0, -1.0},
+ std::vector<double>{-1.0, -1.0, -1.0},
+ std::vector<double>{4.0, 2.0, 1.0},
+ std::vector<double>{9999.0, -1.0, 1.0},
+ std::vector<double>{3.99, 2.01, 1.0},
+ std::vector<double>{4.9, 1.7, 1.25},
+ std::vector<double>{10.0, 4.0, 3.0},
+ std::vector<double>{1.75, 3.5},
+ std::vector<double>{1.5, 2.5},
+ std::vector<double>{1.3, 1.0})));
+
+TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
+ // Set requested resolution alignment.
+ video_source_.set_adaptation_enabled(true);
+ fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
+ fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
+
+ // Fill config with the scaling factor by which to reduce encoding size.
+ const int num_streams = scale_factors_.size();
+ VideoEncoderConfig config;
+ webrtc::VideoEncoder::EncoderInfo encoder_info;
+ test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
+ for (int i = 0; i < num_streams; ++i) {
+ config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
+ }
+ config.video_stream_factory =
+ rtc::make_ref_counted<cricket::EncoderStreamFactory>(
+ "VP8", /*max qp*/ 56, /*screencast*/ false,
+ /*screenshare enabled*/ false, encoder_info);
+ video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
+ 0, 0, 0);
+ // Wait for all layers before triggering event.
+ sink_.SetNumExpectedLayers(num_streams);
+
+ // On the 1st frame, we should have initialized the encoder and
+ // asked for its resolution requirements.
+ int64_t timestamp_ms = kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
+
+ // On the 2nd frame, we should be receiving a correctly aligned resolution.
+ // (It's up the to the encoder to potentially drop the previous frame,
+ // to avoid coding back-to-back keyframes.)
+ timestamp_ms += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
+
+ VideoCodec codec = fake_encoder_.config();
+ EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
+ // Frame size should be a multiple of the requested alignment.
+ for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
+ EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0u);
+ EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0u);
+ // Aspect ratio should match.
+ EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
+ codec.height * codec.simulcastStream[i].width);
+ }
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
+ const int kFramerateFps = 30;
+ const int kWidth = 1280;
+ const int kHeight = 720;
+
+ // We rely on the automatic resolution adaptation, but we handle framerate
+ // adaptation manually by mocking the stats proxy.
+ video_source_.set_adaptation_enabled(true);
+
+ // Enable BALANCED preference, no initial limitation.
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_stream_encoder_->SetSource(&video_source_,
+ webrtc::DegradationPreference::BALANCED);
+ EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Adapt down as far as possible.
+ rtc::VideoSinkWants last_wants;
+ int64_t t = 1;
+ int loop_count = 0;
+ do {
+ ++loop_count;
+ last_wants = video_source_.sink_wants();
+
+ // Simulate the framerate we've been asked to adapt to.
+ const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
+ const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
+ VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
+ mock_stats.input_frame_rate = fps;
+ stats_proxy_->SetMockStats(mock_stats);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(t);
+ t += frame_interval_ms;
+
+ video_stream_encoder_->TriggerCpuOveruse();
+ EXPECT_THAT(
+ video_source_.sink_wants(),
+ FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
+ *video_source_.last_sent_height()));
+ } while (video_source_.sink_wants().max_pixel_count <
+ last_wants.max_pixel_count ||
+ video_source_.sink_wants().max_framerate_fps <
+ last_wants.max_framerate_fps);
+
+ // Verify that we've adapted all the way down.
+ stats_proxy_->ResetMockStats();
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(loop_count - 1,
+ stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
+ *video_source_.last_sent_height());
+ EXPECT_EQ(kMinBalancedFramerateFps,
+ video_source_.sink_wants().max_framerate_fps);
+
+ // Adapt back up the same number of times we adapted down.
+ for (int i = 0; i < loop_count - 1; ++i) {
+ last_wants = video_source_.sink_wants();
+
+ // Simulate the framerate we've been asked to adapt to.
+ const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
+ const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
+ VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
+ mock_stats.input_frame_rate = fps;
+ stats_proxy_->SetMockStats(mock_stats);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(t);
+ t += frame_interval_ms;
+
+ video_stream_encoder_->TriggerCpuUnderuse();
+ EXPECT_THAT(
+ video_source_.sink_wants(),
+ FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
+ *video_source_.last_sent_height()));
+ EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
+ last_wants.max_pixel_count ||
+ video_source_.sink_wants().max_framerate_fps >
+ last_wants.max_framerate_fps);
+ }
+
+ EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
+ stats_proxy_->ResetMockStats();
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ((loop_count - 1) * 2,
+ stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
+ video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
+ kTargetBitrate, 0, 0, 0);
+ EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
+
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+
+ int64_t ntp_time = kFrameIntervalMs;
+
+ // Force an input frame rate to be available, or the adaptation call won't
+ // know what framerate to adapt form.
+ const int kInputFps = 30;
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ stats.input_frame_rate = kInputFps;
+ stats_proxy_->SetMockStats(stats);
+
+ video_source_.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(
+ &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+ EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
+ sink_.WaitForEncodedFrame(ntp_time);
+ ntp_time += kFrameIntervalMs;
+
+ // Trigger CPU overuse.
+ video_stream_encoder_->TriggerCpuOveruse();
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
+ sink_.WaitForEncodedFrame(ntp_time);
+ ntp_time += kFrameIntervalMs;
+
+ EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
+ EXPECT_EQ(std::numeric_limits<int>::max(),
+ video_source_.sink_wants().max_pixel_count);
+ // Some framerate constraint should be set.
+ int restricted_fps = video_source_.sink_wants().max_framerate_fps;
+ EXPECT_LT(restricted_fps, kInputFps);
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
+ sink_.WaitForEncodedFrame(ntp_time);
+ ntp_time += 100;
+
+ video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
+ &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ // Give the encoder queue time to process the change in degradation preference
+ // by waiting for an encoded frame.
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
+ sink_.WaitForEncodedFrame(ntp_time);
+ ntp_time += kFrameIntervalMs;
+
+ video_stream_encoder_->TriggerQualityLow();
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
+ sink_.WaitForEncodedFrame(ntp_time);
+ ntp_time += kFrameIntervalMs;
+
+ // Some resolution constraint should be set.
+ EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
+ EXPECT_LT(video_source_.sink_wants().max_pixel_count,
+ kFrameWidth * kFrameHeight);
+ EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
+
+ int pixel_count = video_source_.sink_wants().max_pixel_count;
+ // Triggering a CPU underuse should not change the sink wants since it has
+ // not been overused for resolution since we changed degradation preference.
+ video_stream_encoder_->TriggerCpuUnderuse();
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
+ sink_.WaitForEncodedFrame(ntp_time);
+ ntp_time += kFrameIntervalMs;
+ EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
+ EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
+
+ // Change the degradation preference back. CPU underuse should not adapt since
+ // QP is most limited.
+ video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
+ &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
+ sink_.WaitForEncodedFrame(ntp_time);
+ ntp_time += 100;
+ // Resolution adaptations is gone after changing degradation preference.
+ EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
+ EXPECT_EQ(std::numeric_limits<int>::max(),
+ video_source_.sink_wants().max_pixel_count);
+ // The fps adaptation from above is now back.
+ EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
+
+ // Trigger CPU underuse.
+ video_stream_encoder_->TriggerCpuUnderuse();
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
+ sink_.WaitForEncodedFrame(ntp_time);
+ ntp_time += kFrameIntervalMs;
+ EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
+
+ // Trigger QP underuse, fps should return to normal.
+ video_stream_encoder_->TriggerQualityHigh();
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
+ sink_.WaitForEncodedFrame(ntp_time);
+ ntp_time += kFrameIntervalMs;
+ EXPECT_THAT(video_source_.sink_wants(), FpsMax());
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
+
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+
+ int64_t frame_timestamp = 1;
+
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(frame_timestamp);
+ frame_timestamp += kFrameIntervalMs;
+
+ // Trigger CPU overuse.
+ video_stream_encoder_->TriggerCpuOveruse();
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(frame_timestamp);
+ frame_timestamp += kFrameIntervalMs;
+
+ // Default degradation preference is maintain-framerate, so will lower max
+ // wanted resolution.
+ EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
+ EXPECT_LT(video_source_.sink_wants().max_pixel_count,
+ kFrameWidth * kFrameHeight);
+ EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
+
+ // Set new source, switch to maintain-resolution.
+ test::FrameForwarder new_video_source;
+ video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
+ &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+ // Give the encoder queue time to process the change in degradation preference
+ // by waiting for an encoded frame.
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
+ sink_.WaitForEncodedFrame(frame_timestamp);
+ frame_timestamp += kFrameIntervalMs;
+ // Initially no degradation registered.
+ EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
+
+ // Force an input frame rate to be available, or the adaptation call won't
+ // know what framerate to adapt form.
+ const int kInputFps = 30;
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ stats.input_frame_rate = kInputFps;
+ stats_proxy_->SetMockStats(stats);
+
+ video_stream_encoder_->TriggerCpuOveruse();
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(frame_timestamp);
+ frame_timestamp += kFrameIntervalMs;
+
+ // Some framerate constraint should be set.
+ EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
+ EXPECT_EQ(std::numeric_limits<int>::max(),
+ new_video_source.sink_wants().max_pixel_count);
+ EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
+
+ // Turn off degradation completely.
+ video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
+ &new_video_source, webrtc::DegradationPreference::DISABLED);
+ // Give the encoder queue time to process the change in degradation preference
+ // by waiting for an encoded frame.
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
+ sink_.WaitForEncodedFrame(frame_timestamp);
+ frame_timestamp += kFrameIntervalMs;
+ EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
+
+ video_stream_encoder_->TriggerCpuOveruse();
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(frame_timestamp);
+ frame_timestamp += kFrameIntervalMs;
+
+ // Still no degradation.
+ EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
+
+ // Calling SetSource with resolution scaling enabled apply the old SinkWants.
+ video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
+ &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ // Give the encoder queue time to process the change in degradation preference
+ // by waiting for an encoded frame.
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
+ sink_.WaitForEncodedFrame(frame_timestamp);
+ frame_timestamp += kFrameIntervalMs;
+ EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
+ kFrameWidth * kFrameHeight);
+ EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
+ EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
+
+ // Calling SetSource with framerate scaling enabled apply the old SinkWants.
+ video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
+ &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+ // Give the encoder queue time to process the change in degradation preference
+ // by waiting for an encoded frame.
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
+ sink_.WaitForEncodedFrame(frame_timestamp);
+ frame_timestamp += kFrameIntervalMs;
+ EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
+ EXPECT_EQ(std::numeric_limits<int>::max(),
+ new_video_source.sink_wants().max_pixel_count);
+ EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ WaitForEncodedFrame(1);
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
+ EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
+
+ // Trigger adapt down.
+ video_stream_encoder_->TriggerQualityLow();
+ video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+ WaitForEncodedFrame(2);
+
+ stats = stats_proxy_->GetStats();
+ EXPECT_TRUE(stats.bw_limited_resolution);
+ EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
+
+ // Trigger adapt up.
+ video_stream_encoder_->TriggerQualityHigh();
+ video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+ WaitForEncodedFrame(3);
+
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
+ EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
+ EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ WaitForEncodedFrame(1);
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
+
+ // Trigger CPU overuse.
+ video_stream_encoder_->TriggerCpuOveruse();
+ video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+ WaitForEncodedFrame(2);
+
+ stats = stats_proxy_->GetStats();
+ EXPECT_TRUE(stats.cpu_limited_resolution);
+ EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
+
+ // Trigger CPU normal use.
+ video_stream_encoder_->TriggerCpuUnderuse();
+ video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+ WaitForEncodedFrame(3);
+
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
+ EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ WaitForEncodedFrame(1);
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
+
+ // Trigger CPU overuse.
+ video_stream_encoder_->TriggerCpuOveruse();
+ video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+ WaitForEncodedFrame(2);
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
+ EXPECT_TRUE(stats.cpu_limited_resolution);
+ EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
+
+ // Set new source with adaptation still enabled.
+ test::FrameForwarder new_video_source;
+ video_stream_encoder_->SetSource(
+ &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+
+ new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+ WaitForEncodedFrame(3);
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
+ EXPECT_TRUE(stats.cpu_limited_resolution);
+ EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
+
+ // Set adaptation disabled.
+ video_stream_encoder_->SetSource(&new_video_source,
+ webrtc::DegradationPreference::DISABLED);
+
+ new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
+ WaitForEncodedFrame(4);
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
+
+ // Set adaptation back to enabled.
+ video_stream_encoder_->SetSource(
+ &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+
+ new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
+ WaitForEncodedFrame(5);
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
+ EXPECT_TRUE(stats.cpu_limited_resolution);
+ EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
+
+ // Trigger CPU normal use.
+ video_stream_encoder_->TriggerCpuUnderuse();
+ new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
+ WaitForEncodedFrame(6);
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
+ EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ WaitForEncodedFrame(1);
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
+ EXPECT_FALSE(stats.bw_limited_framerate);
+ EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
+
+ // Set new source with adaptation still enabled.
+ test::FrameForwarder new_video_source;
+ video_stream_encoder_->SetSource(&new_video_source,
+ webrtc::DegradationPreference::BALANCED);
+
+ new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+ WaitForEncodedFrame(2);
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
+ EXPECT_FALSE(stats.bw_limited_framerate);
+ EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
+
+ // Trigger adapt down.
+ video_stream_encoder_->TriggerQualityLow();
+ new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+ WaitForEncodedFrame(3);
+ stats = stats_proxy_->GetStats();
+ EXPECT_TRUE(stats.bw_limited_resolution);
+ EXPECT_FALSE(stats.bw_limited_framerate);
+ EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
+
+ // Set new source with adaptation still enabled.
+ video_stream_encoder_->SetSource(&new_video_source,
+ webrtc::DegradationPreference::BALANCED);
+
+ new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
+ WaitForEncodedFrame(4);
+ stats = stats_proxy_->GetStats();
+ EXPECT_TRUE(stats.bw_limited_resolution);
+ EXPECT_FALSE(stats.bw_limited_framerate);
+ EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
+
+ // Disable resolution scaling.
+ video_stream_encoder_->SetSource(
+ &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+
+ new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
+ WaitForEncodedFrame(5);
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.bw_limited_resolution);
+ EXPECT_FALSE(stats.bw_limited_framerate);
+ EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
+ EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ int64_t timestamp_ms = kFrameIntervalMs;
+ video_source_.set_adaptation_enabled(true);
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Trigger adapt down.
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Trigger overuse.
+ video_stream_encoder_->TriggerCpuOveruse();
+ timestamp_ms += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Leave source unchanged, but disable quality scaler.
+ fake_encoder_.SetQualityScaling(false);
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
+ // Make format different, to force recreation of encoder.
+ video_encoder_config.video_format.parameters["foo"] = "foo";
+ video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
+ kMaxPayloadLength);
+ timestamp_ms += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ int sequence = 1;
+
+ // Enable BALANCED preference, no initial limitation.
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(&source,
+ webrtc::DegradationPreference::BALANCED);
+ source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
+ EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
+
+ // Trigger CPU overuse, should now adapt down.
+ video_stream_encoder_->TriggerCpuOveruse();
+ source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ stats = stats_proxy_->GetStats();
+ EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
+
+ // Set new degradation preference should clear restrictions since we changed
+ // from BALANCED.
+ video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
+ EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
+
+ // Force an input frame rate to be available, or the adaptation call won't
+ // know what framerate to adapt from.
+ VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
+ mock_stats.input_frame_rate = 30;
+ stats_proxy_->SetMockStats(mock_stats);
+ video_stream_encoder_->TriggerCpuOveruse();
+ stats_proxy_->ResetMockStats();
+ source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+
+ // We have now adapted once.
+ stats = stats_proxy_->GetStats();
+ EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
+
+ // Back to BALANCED, should clear the restrictions again.
+ video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
+ &source, webrtc::DegradationPreference::BALANCED);
+ source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
+ EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ int sequence = 1;
+
+ video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
+ EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
+
+ // Trigger CPU overuse, should now adapt down.
+ video_stream_encoder_->TriggerCpuOveruse();
+ video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ stats = stats_proxy_->GetStats();
+ EXPECT_TRUE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
+ EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
+
+ // Set new source with adaptation still enabled.
+ test::FrameForwarder new_video_source;
+ video_stream_encoder_->SetSource(
+ &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ stats = stats_proxy_->GetStats();
+ EXPECT_TRUE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
+ EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
+
+ // Set cpu adaptation by frame dropping.
+ video_stream_encoder_->SetSource(
+ &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ stats = stats_proxy_->GetStats();
+ // Not adapted at first.
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
+ EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
+
+ // Force an input frame rate to be available, or the adaptation call won't
+ // know what framerate to adapt from.
+ VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
+ mock_stats.input_frame_rate = 30;
+ stats_proxy_->SetMockStats(mock_stats);
+ video_stream_encoder_->TriggerCpuOveruse();
+ stats_proxy_->ResetMockStats();
+
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+
+ // Framerate now adapted.
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_TRUE(stats.cpu_limited_framerate);
+ EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
+
+ // Disable CPU adaptation.
+ video_stream_encoder_->SetSource(&new_video_source,
+ webrtc::DegradationPreference::DISABLED);
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
+ EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
+
+ // Try to trigger overuse. Should not succeed.
+ stats_proxy_->SetMockStats(mock_stats);
+ video_stream_encoder_->TriggerCpuOveruse();
+ stats_proxy_->ResetMockStats();
+
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
+ EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
+
+ // Switch back the source with resolution adaptation enabled.
+ video_stream_encoder_->SetSource(
+ &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ stats = stats_proxy_->GetStats();
+ EXPECT_TRUE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
+ EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
+
+ // Trigger CPU normal usage.
+ video_stream_encoder_->TriggerCpuUnderuse();
+ video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
+ EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
+
+ // Back to the source with adaptation off, set it back to maintain-resolution.
+ video_stream_encoder_->SetSource(
+ &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ stats = stats_proxy_->GetStats();
+ // Disabled, since we previously switched the source to disabled.
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_TRUE(stats.cpu_limited_framerate);
+ EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
+
+ // Trigger CPU normal usage.
+ video_stream_encoder_->TriggerCpuUnderuse();
+ new_video_source.IncomingCapturedFrame(
+ CreateFrame(sequence, kWidth, kHeight));
+ WaitForEncodedFrame(sequence++);
+ stats = stats_proxy_->GetStats();
+ EXPECT_FALSE(stats.cpu_limited_resolution);
+ EXPECT_FALSE(stats.cpu_limited_framerate);
+ EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
+ EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ ScalingUpAndDownDoesNothingWithMaintainResolution) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Expect no scaling to begin with.
+ EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ WaitForEncodedFrame(1);
+
+ // Trigger scale down.
+ video_stream_encoder_->TriggerQualityLow();
+
+ video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+ WaitForEncodedFrame(2);
+
+ // Expect a scale down.
+ EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
+ EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
+
+ // Set resolution scaling disabled.
+ test::FrameForwarder new_video_source;
+ video_stream_encoder_->SetSource(
+ &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+
+ // Trigger scale down.
+ video_stream_encoder_->TriggerQualityLow();
+ new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+ WaitForEncodedFrame(3);
+
+ // Expect no scaling.
+ EXPECT_EQ(std::numeric_limits<int>::max(),
+ new_video_source.sink_wants().max_pixel_count);
+
+ // Trigger scale up.
+ video_stream_encoder_->TriggerQualityHigh();
+ new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
+ WaitForEncodedFrame(4);
+
+ // Expect nothing to change, still no scaling.
+ EXPECT_EQ(std::numeric_limits<int>::max(),
+ new_video_source.sink_wants().max_pixel_count);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ SkipsSameAdaptDownRequest_MaintainFramerateMode) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+
+ source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ WaitForEncodedFrame(1);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Trigger adapt down, expect scaled down resolution.
+ video_stream_encoder_->TriggerCpuOveruse();
+ EXPECT_THAT(source.sink_wants(),
+ FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
+ const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Trigger adapt down for same input resolution, expect no change.
+ video_stream_encoder_->TriggerCpuOveruse();
+ EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable BALANCED preference, no initial limitation.
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(&source,
+ webrtc::DegradationPreference::BALANCED);
+ source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(1);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+
+ // Trigger adapt down, expect scaled down resolution.
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_THAT(source.sink_wants(),
+ FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+ const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
+
+ // Trigger adapt down for same input resolution, expect no change.
+ source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(2);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down for larger input resolution, expect no change.
+ source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
+ sink_.WaitForEncodedFrame(3);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
+ const int kWidth = 640;
+ const int kHeight = 360;
+ const int64_t kFrameIntervalMs = 150;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable BALANCED preference, no initial limitation.
+ AdaptingFrameForwarder source(&time_controller_);
+ source.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(&source,
+ webrtc::DegradationPreference::BALANCED);
+
+ int64_t timestamp_ms = kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect reduced fps (640x360@15fps).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(),
+ FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Source requests 270p, expect reduced resolution (480x270@15fps).
+ source.OnOutputFormatRequest(480, 270);
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(480, 270);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect reduced fps (480x270@10fps).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Source requests QVGA, expect reduced resolution (320x180@10fps).
+ source.OnOutputFormatRequest(320, 180);
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(320, 180);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect reduced fps (320x180@7fps).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Source requests VGA, expect increased resolution (640x360@7fps).
+ source.OnOutputFormatRequest(640, 360);
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect increased fps (640x360@maxfps).
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ const int64_t kFrameIntervalMs = 150;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable BALANCED preference, no initial limitation.
+ AdaptingFrameForwarder source(&time_controller_);
+ source.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(&source,
+ webrtc::DegradationPreference::BALANCED);
+
+ int64_t timestamp_ms = kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(),
+ FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect reduced fps (640x360@15fps).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Source requests QVGA, expect reduced resolution (320x180@15fps).
+ source.OnOutputFormatRequest(320, 180);
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(320, 180);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Trigger adapt down, expect reduced fps (320x180@7fps).
+ video_stream_encoder_->TriggerCpuOveruse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Source requests HD, expect increased resolution (640x360@7fps).
+ source.OnOutputFormatRequest(1280, 720);
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
+ video_stream_encoder_->TriggerCpuUnderuse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Trigger adapt up, expect increased fps (640x360@maxfps).
+ video_stream_encoder_->TriggerQualityHigh();
+ video_stream_encoder_->TriggerCpuUnderuse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Trigger adapt up, expect increased resolution (960x570@maxfps).
+ video_stream_encoder_->TriggerQualityHigh();
+ video_stream_encoder_->TriggerCpuUnderuse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Trigger adapt up, expect increased resolution (1280x720@maxfps).
+ video_stream_encoder_->TriggerQualityHigh();
+ video_stream_encoder_->TriggerCpuUnderuse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+ EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ NoChangeForInitialNormalUsage_MaintainFramerateMode) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+
+ source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Trigger adapt up, expect no change.
+ video_stream_encoder_->TriggerCpuUnderuse();
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ NoChangeForInitialNormalUsage_MaintainResolutionMode) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+
+ source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Trigger adapt up, expect no change.
+ video_stream_encoder_->TriggerCpuUnderuse();
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable BALANCED preference, no initial limitation.
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(&source,
+ webrtc::DegradationPreference::BALANCED);
+
+ source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect no change.
+ video_stream_encoder_->TriggerQualityHigh();
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable DISABLED preference, no initial limitation.
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(&source,
+ webrtc::DegradationPreference::DISABLED);
+
+ source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect no change.
+ video_stream_encoder_->TriggerQualityHigh();
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ AdaptsResolutionForLowQuality_MaintainFramerateMode) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
+ AdaptingFrameForwarder source(&time_controller_);
+ source.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+
+ source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ WaitForEncodedFrame(1);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down resolution.
+ video_stream_encoder_->TriggerQualityLow();
+ source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+ WaitForEncodedFrame(2);
+ EXPECT_THAT(source.sink_wants(),
+ FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect no restriction.
+ video_stream_encoder_->TriggerQualityHigh();
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ AdaptsFramerateForLowQuality_MaintainResolutionMode) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ const int kInputFps = 30;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ stats.input_frame_rate = kInputFps;
+ stats_proxy_->SetMockStats(stats);
+
+ // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(1);
+ EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
+
+ // Trigger adapt down, expect scaled down resolution.
+ video_stream_encoder_->TriggerQualityLow();
+ video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(2);
+ EXPECT_THAT(video_source_.sink_wants(),
+ FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
+
+ // Enable MAINTAIN_RESOLUTION preference.
+ test::FrameForwarder new_video_source;
+ video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
+ &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+ // Give the encoder queue time to process the change in degradation preference
+ // by waiting for an encoded frame.
+ new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(3);
+ EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
+
+ // Trigger adapt down, expect reduced framerate.
+ video_stream_encoder_->TriggerQualityLow();
+ new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(4);
+ EXPECT_THAT(new_video_source.sink_wants(),
+ FpsMatchesResolutionMax(Lt(kInputFps)));
+
+ // Trigger adapt up, expect no restriction.
+ video_stream_encoder_->TriggerQualityHigh();
+ EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ const size_t kNumFrames = 10;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable adapter, expected input resolutions when downscaling:
+ // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
+ video_source_.set_adaptation_enabled(true);
+
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ int downscales = 0;
+ for (size_t i = 1; i <= kNumFrames; i++) {
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
+ WaitForEncodedFrame(i * kFrameIntervalMs);
+
+ // Trigger scale down.
+ rtc::VideoSinkWants last_wants = video_source_.sink_wants();
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
+
+ if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
+ ++downscales;
+
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(downscales,
+ stats_proxy_->GetStats().number_of_quality_adapt_changes);
+ EXPECT_GT(downscales, 0);
+ }
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
+ AdaptingFrameForwarder source(&time_controller_);
+ source.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+
+ int64_t timestamp_ms = kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Trigger adapt down, expect scaled down resolution.
+ video_stream_encoder_->TriggerCpuOveruse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(),
+ FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Trigger adapt up, expect no restriction.
+ video_stream_encoder_->TriggerCpuUnderuse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Trigger adapt down, expect scaled down resolution.
+ video_stream_encoder_->TriggerCpuOveruse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(),
+ FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ // Trigger adapt up, expect no restriction.
+ video_stream_encoder_->TriggerCpuUnderuse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable BALANCED preference, no initial limitation.
+ AdaptingFrameForwarder source(&time_controller_);
+ source.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(&source,
+ webrtc::DegradationPreference::BALANCED);
+
+ int64_t timestamp_ms = kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down resolution.
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(),
+ FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect no restriction.
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down resolution.
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(),
+ FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect no restriction.
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
+ fake_encoder_.SetResolutionBitrateLimits(
+ {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
+ DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
+ DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
+ 0, 0);
+
+ // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
+ AdaptingFrameForwarder source(&time_controller_);
+ source.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+
+ // Insert 720p frame.
+ int64_t timestamp_ms = kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
+ WaitForEncodedFrame(1280, 720);
+
+ // Reduce bitrate and trigger adapt down.
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
+ DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
+ DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
+ 0, 0);
+ video_stream_encoder_->TriggerQualityLow();
+
+ // Insert 720p frame. It should be downscaled and encoded.
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
+ WaitForEncodedFrame(960, 540);
+
+ // Trigger adapt up. Higher resolution should not be requested duo to lack
+ // of bitrate.
+ video_stream_encoder_->TriggerQualityHigh();
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
+
+ // Increase bitrate.
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
+ DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
+ DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
+ 0, 0);
+
+ // Trigger adapt up. Higher resolution should be requested.
+ video_stream_encoder_->TriggerQualityHigh();
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
+ fake_encoder_.SetResolutionBitrateLimits(
+ {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
+
+ // Set bitrate equal to min bitrate of 540p.
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
+ DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
+ DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
+ 0, 0);
+
+ // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
+ AdaptingFrameForwarder source(&time_controller_);
+ source.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+
+ // Insert 720p frame. It should be dropped and lower resolution should be
+ // requested.
+ int64_t timestamp_ms = kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
+ ExpectDroppedFrame();
+ EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
+
+ // Insert 720p frame. It should be downscaled and encoded.
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
+ WaitForEncodedFrame(960, 540);
+
+ video_stream_encoder_->Stop();
+}
+
+class BalancedDegradationTest : public VideoStreamEncoderTest {
+ protected:
+ void SetupTest() {
+ // Reset encoder for field trials to take effect.
+ ConfigureEncoder(video_encoder_config_.Copy());
+ OnBitrateUpdated(kTargetBitrate);
+
+ // Enable BALANCED preference.
+ source_.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
+ }
+
+ void OnBitrateUpdated(DataRate bitrate) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ bitrate, bitrate, bitrate, 0, 0, 0);
+ }
+
+ void InsertFrame() {
+ timestamp_ms_ += kFrameIntervalMs;
+ source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
+ }
+
+ void InsertFrameAndWaitForEncoded() {
+ InsertFrame();
+ sink_.WaitForEncodedFrame(timestamp_ms_);
+ }
+
+ const int kWidth = 640; // pixels:640x360=230400
+ const int kHeight = 360;
+ const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
+ int64_t timestamp_ms_ = 0;
+ AdaptingFrameForwarder source_{&time_controller_};
+};
+
+TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
+ test::ScopedKeyValueConfig field_trials(
+ field_trials_,
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
+ SetupTest();
+
+ // Force input frame rate.
+ const int kInputFps = 24;
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ stats.input_frame_rate = kInputFps;
+ stats_proxy_->SetMockStats(stats);
+
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
+
+ // Trigger adapt down, expect scaled down framerate and resolution,
+ // since Fps diff (input-requested:0) < threshold.
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_THAT(source_.sink_wants(),
+ AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
+ test::ScopedKeyValueConfig field_trials(
+ field_trials_,
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
+ SetupTest();
+
+ // Force input frame rate.
+ const int kInputFps = 25;
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ stats.input_frame_rate = kInputFps;
+ stats_proxy_->SetMockStats(stats);
+
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
+
+ // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
+ // Fps diff (input-requested:1) == threshold.
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
+ test::ScopedKeyValueConfig field_trials(
+ field_trials_,
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
+ SetupTest();
+
+ EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
+
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
+
+ // Trigger adapt down, expect scaled down framerate (640x360@22fps).
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
+ test::ScopedKeyValueConfig field_trials(
+ field_trials_,
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
+ SetupTest();
+
+ const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
+ const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
+ OnBitrateUpdated(kTooLowMinBitrate);
+
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down framerate (640x360@14fps).
+ video_stream_encoder_->TriggerQualityLow();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down resolution (480x270@14fps).
+ video_stream_encoder_->TriggerQualityLow();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down framerate (480x270@10fps).
+ video_stream_encoder_->TriggerQualityLow();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
+ EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
+ video_stream_encoder_->TriggerQualityHigh();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
+ OnBitrateUpdated(kMinBitrate);
+ video_stream_encoder_->TriggerQualityHigh();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(BalancedDegradationTest,
+ InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
+ test::ScopedKeyValueConfig field_trials(
+ field_trials_,
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:57600|129600|230400,fps:7|24|24/");
+ SetupTest();
+ OnBitrateUpdated(kLowTargetBitrate);
+
+ EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
+
+ // Insert frame, expect scaled down:
+ // framerate (640x360@24fps) -> resolution (480x270@24fps).
+ InsertFrame();
+ EXPECT_FALSE(WaitForFrame(TimeDelta::Seconds(1)));
+ EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
+ EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
+
+ // Insert frame, expect scaled down:
+ // resolution (320x180@24fps).
+ InsertFrame();
+ EXPECT_FALSE(WaitForFrame(TimeDelta::Seconds(1)));
+ EXPECT_LT(source_.sink_wants().max_pixel_count,
+ source_.last_wants().max_pixel_count);
+ EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
+
+ // Frame should not be dropped (min pixels per frame reached).
+ InsertFrameAndWaitForEncoded();
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(BalancedDegradationTest,
+ NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
+ test::ScopedKeyValueConfig field_trials(
+ field_trials_,
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
+ SetupTest();
+
+ const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
+ const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
+ OnBitrateUpdated(kTooLowMinResolutionBitrate);
+
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down framerate (640x360@14fps).
+ video_stream_encoder_->TriggerQualityLow();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down resolution (480x270@14fps).
+ video_stream_encoder_->TriggerQualityLow();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down framerate (480x270@10fps).
+ video_stream_encoder_->TriggerQualityLow();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
+ video_stream_encoder_->TriggerQualityHigh();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
+ video_stream_encoder_->TriggerQualityHigh();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
+ OnBitrateUpdated(kResolutionMinBitrate);
+ video_stream_encoder_->TriggerQualityHigh();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
+ EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(BalancedDegradationTest,
+ NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
+ test::ScopedKeyValueConfig field_trials(
+ field_trials_,
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
+ SetupTest();
+
+ const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
+ const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
+ const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
+ const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
+ OnBitrateUpdated(kTooLowMinBitrate);
+
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down framerate (640x360@14fps).
+ video_stream_encoder_->TriggerQualityLow();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down resolution (480x270@14fps).
+ video_stream_encoder_->TriggerQualityLow();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down framerate (480x270@10fps).
+ video_stream_encoder_->TriggerQualityLow();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
+ video_stream_encoder_->TriggerQualityHigh();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
+ OnBitrateUpdated(kMinBitrate);
+ video_stream_encoder_->TriggerQualityHigh();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
+ OnBitrateUpdated(kTooLowMinResolutionBitrate);
+ video_stream_encoder_->TriggerQualityHigh();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
+ OnBitrateUpdated(kResolutionMinBitrate);
+ video_stream_encoder_->TriggerQualityHigh();
+ InsertFrameAndWaitForEncoded();
+ EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
+ EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
+ AdaptingFrameForwarder source(&time_controller_);
+ source.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+
+ int64_t timestamp_ms = kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger cpu adapt down, expect scaled down resolution (960x540).
+ video_stream_encoder_->TriggerCpuOveruse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(),
+ FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger cpu adapt down, expect scaled down resolution (640x360).
+ video_stream_encoder_->TriggerCpuOveruse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger cpu adapt down, expect scaled down resolution (480x270).
+ video_stream_encoder_->TriggerCpuOveruse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger quality adapt down, expect scaled down resolution (320x180).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
+ rtc::VideoSinkWants last_wants = source.sink_wants();
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger quality adapt down, expect no change (min resolution reached).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsMax());
+ EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger quality adapt up, expect upscaled resolution (480x270).
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger quality and cpu adapt up since both are most limited, expect
+ // upscaled resolution (640x360).
+ video_stream_encoder_->TriggerCpuUnderuse();
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger quality and cpu adapt up since both are most limited, expect
+ // upscaled resolution (960x540).
+ video_stream_encoder_->TriggerCpuUnderuse();
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
+ last_wants = source.sink_wants();
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger cpu adapt up, expect no change since not most limited (960x540).
+ // However the stats will change since the CPU resource is no longer limited.
+ video_stream_encoder_->TriggerCpuUnderuse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger quality adapt up, expect no restriction (1280x720).
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
+ const int kWidth = 640;
+ const int kHeight = 360;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
+ video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
+ WaitForEncodedFrame(i);
+ }
+
+ video_stream_encoder_->TriggerCpuOveruse();
+ for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
+ video_source_.IncomingCapturedFrame(CreateFrame(
+ SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
+ WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
+ }
+
+ video_stream_encoder_->Stop();
+ video_stream_encoder_.reset();
+ stats_proxy_.reset();
+
+ EXPECT_METRIC_EQ(
+ 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
+ EXPECT_METRIC_EQ(
+ 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
+}
+
+TEST_F(VideoStreamEncoderTest,
+ CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ const int kWidth = 640;
+ const int kHeight = 360;
+
+ video_stream_encoder_->SetSource(&video_source_,
+ webrtc::DegradationPreference::DISABLED);
+
+ for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
+ video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
+ WaitForEncodedFrame(i);
+ }
+
+ video_stream_encoder_->Stop();
+ video_stream_encoder_.reset();
+ stats_proxy_.reset();
+
+ EXPECT_EQ(0,
+ metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
+}
+
+TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
+ ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoBitrateAllocation);
+
+ const int kDefaultFps = 30;
+ const VideoBitrateAllocation expected_bitrate =
+ SimulcastRateAllocator(fake_encoder_.config())
+ .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
+ kDefaultFps));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
+ EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
+
+ // Check that encoder has been updated too, not just allocation observer.
+ EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
+ AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
+
+ // VideoBitrateAllocation not updated on second frame.
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
+ AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
+
+ // VideoBitrateAllocation updated after a process interval.
+ const int64_t start_time_ms = CurrentTimeMs();
+ while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+ WaitForEncodedFrame(CurrentTimeMs());
+ AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
+ }
+ EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
+ ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoLayersAllocation);
+
+ const int kDefaultFps = 30;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+ VideoLayersAllocation last_layer_allocation =
+ sink_.GetLastVideoLayersAllocation();
+ // kLowTargetBitrate is only enough for one spatial layer.
+ ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
+
+ VideoBitrateAllocation bitrate_allocation =
+ fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
+ // Check that encoder has been updated too, not just allocation observer.
+ EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
+ AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
+
+ // VideoLayersAllocation might be updated if frame rate changes.
+ int number_of_layers_allocation = 1;
+ const int64_t start_time_ms = CurrentTimeMs();
+ while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+ WaitForEncodedFrame(CurrentTimeMs());
+ if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
+ number_of_layers_allocation = sink_.number_of_layers_allocations();
+ VideoLayersAllocation new_allocation =
+ sink_.GetLastVideoLayersAllocation();
+ ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
+ EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
+ last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
+ EXPECT_EQ(new_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer,
+ last_layer_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer);
+ last_layer_allocation = new_allocation;
+ }
+ }
+ EXPECT_LE(sink_.number_of_layers_allocations(), 3);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
+ /* num_streams*/ 3, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+ video_encoder_config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
+ VideoEncoder::GetDefaultVp8Settings());
+ for (auto& layer : video_encoder_config.simulcast_layers) {
+ layer.num_temporal_layers = 2;
+ }
+ // Simulcast layers are used for enabling/disabling streams.
+ video_encoder_config.simulcast_layers[0].active = true;
+ video_encoder_config.simulcast_layers[1].active = false;
+ video_encoder_config.simulcast_layers[2].active = true;
+ ConfigureEncoder(std::move(video_encoder_config),
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoLayersAllocation);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+ VideoLayersAllocation last_layer_allocation =
+ sink_.GetLastVideoLayersAllocation();
+
+ ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
+ EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer,
+ SizeIs(2));
+ EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
+ /* num_streams*/ 3, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+ video_encoder_config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
+ VideoEncoder::GetDefaultVp8Settings());
+ for (auto& layer : video_encoder_config.simulcast_layers) {
+ layer.num_temporal_layers = 2;
+ }
+ // Simulcast layers are used for enabling/disabling streams.
+ video_encoder_config.simulcast_layers[0].active = true;
+ video_encoder_config.simulcast_layers[1].active = false;
+ video_encoder_config.simulcast_layers[2].active = false;
+ ConfigureEncoder(std::move(video_encoder_config),
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoLayersAllocation);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+ VideoLayersAllocation last_layer_allocation =
+ sink_.GetLastVideoLayersAllocation();
+
+ ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
+ EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer,
+ SizeIs(2));
+ EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
+ /* num_streams*/ 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+ VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
+ vp9_settings.numberOfSpatialLayers = 2;
+ vp9_settings.numberOfTemporalLayers = 2;
+ vp9_settings.interLayerPred = InterLayerPredMode::kOn;
+ vp9_settings.automaticResizeOn = false;
+ video_encoder_config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
+ vp9_settings);
+ ConfigureEncoder(std::move(video_encoder_config),
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoLayersAllocation);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+ VideoLayersAllocation last_layer_allocation =
+ sink_.GetLastVideoLayersAllocation();
+
+ ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
+ EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer,
+ SizeIs(2));
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
+ EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
+ .target_bitrate_per_temporal_layer,
+ SizeIs(2));
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
+
+ // Since full SVC is used, expect the top layer to utilize the full target
+ // rate.
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
+ .target_bitrate_per_temporal_layer[1],
+ kTargetBitrate);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
+ /* num_streams*/ 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+ VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
+ vp9_settings.numberOfSpatialLayers = 2;
+ vp9_settings.numberOfTemporalLayers = 2;
+ vp9_settings.interLayerPred = InterLayerPredMode::kOn;
+ vp9_settings.automaticResizeOn = false;
+ video_encoder_config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
+ vp9_settings);
+ ConfigureEncoder(std::move(video_encoder_config),
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoLayersAllocation);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+ VideoLayersAllocation last_layer_allocation =
+ sink_.GetLastVideoLayersAllocation();
+
+ ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
+ EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer,
+ SizeIs(1));
+ EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
+ .target_bitrate_per_temporal_layer,
+ SizeIs(1));
+ // Since full SVC is used, expect the top layer to utilize the full target
+ // rate.
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
+ .target_bitrate_per_temporal_layer[0],
+ kTargetBitrate);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
+ /* num_streams*/ 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+ VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
+ vp9_settings.numberOfSpatialLayers = 2;
+ vp9_settings.numberOfTemporalLayers = 2;
+ vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
+ vp9_settings.automaticResizeOn = false;
+ video_encoder_config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
+ vp9_settings);
+ ConfigureEncoder(std::move(video_encoder_config),
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoLayersAllocation);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+ VideoLayersAllocation last_layer_allocation =
+ sink_.GetLastVideoLayersAllocation();
+
+ ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
+ EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer,
+ SizeIs(2));
+ EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
+ .target_bitrate_per_temporal_layer,
+ SizeIs(2));
+ // Since KSVC is, spatial layers are independend except on key frames.
+ EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
+ .target_bitrate_per_temporal_layer[1],
+ kTargetBitrate);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
+ /* num_streams*/ 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+ VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
+ vp9_settings.numberOfSpatialLayers = 3;
+ vp9_settings.numberOfTemporalLayers = 2;
+ vp9_settings.interLayerPred = InterLayerPredMode::kOn;
+ vp9_settings.automaticResizeOn = false;
+ video_encoder_config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
+ vp9_settings);
+ // Simulcast layers are used for enabling/disabling streams.
+ video_encoder_config.simulcast_layers.resize(3);
+ video_encoder_config.simulcast_layers[0].active = false;
+ video_encoder_config.simulcast_layers[1].active = true;
+ video_encoder_config.simulcast_layers[2].active = true;
+ ConfigureEncoder(std::move(video_encoder_config),
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoLayersAllocation);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+ VideoLayersAllocation last_layer_allocation =
+ sink_.GetLastVideoLayersAllocation();
+
+ ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
+ EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer,
+ SizeIs(2));
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
+
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
+ EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
+ .target_bitrate_per_temporal_layer,
+ SizeIs(2));
+ // Since full SVC is used, expect the top layer to utilize the full target
+ // rate.
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
+ .target_bitrate_per_temporal_layer[1],
+ kTargetBitrate);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
+ /* num_streams*/ 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+ VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
+ vp9_settings.numberOfSpatialLayers = 3;
+ vp9_settings.numberOfTemporalLayers = 2;
+ vp9_settings.interLayerPred = InterLayerPredMode::kOn;
+ vp9_settings.automaticResizeOn = false;
+ video_encoder_config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
+ vp9_settings);
+ // Simulcast layers are used for enabling/disabling streams.
+ video_encoder_config.simulcast_layers.resize(3);
+ video_encoder_config.simulcast_layers[2].active = false;
+ ConfigureEncoder(std::move(video_encoder_config),
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoLayersAllocation);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+ VideoLayersAllocation last_layer_allocation =
+ sink_.GetLastVideoLayersAllocation();
+
+ ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
+ EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer,
+ SizeIs(2));
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
+
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
+ EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
+ .target_bitrate_per_temporal_layer,
+ SizeIs(2));
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
+ /* num_streams*/ 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+ VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
+ vp9_settings.numberOfSpatialLayers = 3;
+ vp9_settings.numberOfTemporalLayers = 2;
+ vp9_settings.interLayerPred = InterLayerPredMode::kOn;
+ vp9_settings.automaticResizeOn = false;
+ video_encoder_config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
+ vp9_settings);
+ // Simulcast layers are used for enabling/disabling streams.
+ video_encoder_config.simulcast_layers.resize(3);
+ video_encoder_config.simulcast_layers[0].active = false;
+ video_encoder_config.simulcast_layers[1].active = false;
+ video_encoder_config.simulcast_layers[2].active = true;
+ ConfigureEncoder(std::move(video_encoder_config),
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoLayersAllocation);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+ VideoLayersAllocation last_layer_allocation =
+ sink_.GetLastVideoLayersAllocation();
+
+ ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
+ EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer,
+ SizeIs(2));
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer[1],
+ kTargetBitrate);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
+ ResetEncoder("H264", 1, 1, 1, false,
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoLayersAllocation);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+ VideoLayersAllocation last_layer_allocation =
+ sink_.GetLastVideoLayersAllocation();
+
+ ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
+ ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer,
+ SizeIs(1));
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer[0],
+ kTargetBitrate);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
+ ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoLayersAllocation);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+ VideoLayersAllocation last_layer_allocation =
+ sink_.GetLastVideoLayersAllocation();
+ // kLowTargetBitrate is only enough for one spatial layer.
+ ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
+ EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
+ .target_bitrate_per_temporal_layer[0],
+ kLowTargetBitrate);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
+ 0, 0, 0);
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+ WaitForEncodedFrame(CurrentTimeMs());
+
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
+ last_layer_allocation = sink_.GetLastVideoLayersAllocation();
+ ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
+ EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
+ .target_bitrate_per_temporal_layer[0],
+ DataRate::Zero());
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
+ ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoLayersAllocation);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
+ 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+ ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
+ SizeIs(2));
+ EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
+ codec_width_);
+ EXPECT_EQ(
+ sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
+ codec_height_);
+
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
+ ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
+ SizeIs(2));
+ EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
+ codec_width_ / 2);
+ EXPECT_EQ(
+ sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
+ codec_height_ / 2);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
+ // 2 TLs configured, temporal layers supported by encoder.
+ const int kNumTemporalLayers = 2;
+ ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoBitrateAllocation);
+ fake_encoder_.SetTemporalLayersSupported(0, true);
+
+ // Bitrate allocated across temporal layers.
+ const int kTl0Bps = kTargetBitrate.bps() *
+ webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
+ kNumTemporalLayers, /*temporal_id*/ 0,
+ /*base_heavy_tl3_alloc*/ false);
+ const int kTl1Bps = kTargetBitrate.bps() *
+ webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
+ kNumTemporalLayers, /*temporal_id*/ 1,
+ /*base_heavy_tl3_alloc*/ false);
+ VideoBitrateAllocation expected_bitrate;
+ expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
+ expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
+
+ VerifyAllocatedBitrate(expected_bitrate);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
+ // 2 TLs configured, temporal layers not supported by encoder.
+ ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoBitrateAllocation);
+ fake_encoder_.SetTemporalLayersSupported(0, false);
+
+ // Temporal layers not supported by the encoder.
+ // Total bitrate should be at ti:0.
+ VideoBitrateAllocation expected_bitrate;
+ expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
+
+ VerifyAllocatedBitrate(expected_bitrate);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
+ webrtc::test::ScopedKeyValueConfig field_trials(
+ field_trials_,
+ "WebRTC-Video-QualityScalerSettings/"
+ "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
+ // Reset encoder for field trials to take effect.
+ ConfigureEncoder(video_encoder_config_.Copy());
+
+ // 2 TLs configured, temporal layers only supported for first stream.
+ ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoBitrateAllocation);
+ fake_encoder_.SetTemporalLayersSupported(0, true);
+ fake_encoder_.SetTemporalLayersSupported(1, false);
+
+ const int kS0Bps = 150000;
+ const int kS0Tl0Bps =
+ kS0Bps *
+ webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
+ /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
+ const int kS0Tl1Bps =
+ kS0Bps *
+ webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
+ /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
+ const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
+ // Temporal layers not supported by si:1.
+ VideoBitrateAllocation expected_bitrate;
+ expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
+ expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
+ expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
+
+ VerifyAllocatedBitrate(expected_bitrate);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+ const int kFramerate = 24;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+
+ // Insert a single frame, triggering initial configuration.
+ source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ EXPECT_EQ(
+ video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
+ kDefaultFramerate);
+
+ // Trigger reconfigure encoder (without resetting the entire instance).
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
+ video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
+ video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
+ video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
+ kMaxPayloadLength);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ // Detector should be updated with fps limit from codec config.
+ EXPECT_EQ(
+ video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
+ kFramerate);
+
+ // Trigger overuse, max framerate should be reduced.
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ stats.input_frame_rate = kFramerate;
+ stats_proxy_->SetMockStats(stats);
+ video_stream_encoder_->TriggerCpuOveruse();
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ int adapted_framerate =
+ video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
+ EXPECT_LT(adapted_framerate, kFramerate);
+
+ // Trigger underuse, max framerate should go back to codec configured fps.
+ // Set extra low fps, to make sure it's actually reset, not just incremented.
+ stats = stats_proxy_->GetStats();
+ stats.input_frame_rate = adapted_framerate / 2;
+ stats_proxy_->SetMockStats(stats);
+ video_stream_encoder_->TriggerCpuUnderuse();
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(
+ video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
+ kFramerate);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+ const int kLowFramerate = 15;
+ const int kHighFramerate = 25;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+
+ // Trigger initial configuration.
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
+ video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
+ video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
+ source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ EXPECT_EQ(
+ video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
+ kLowFramerate);
+
+ // Trigger overuse, max framerate should be reduced.
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ stats.input_frame_rate = kLowFramerate;
+ stats_proxy_->SetMockStats(stats);
+ video_stream_encoder_->TriggerCpuOveruse();
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ int adapted_framerate =
+ video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
+ EXPECT_LT(adapted_framerate, kLowFramerate);
+
+ // Reconfigure the encoder with a new (higher max framerate), max fps should
+ // still respect the adaptation.
+ video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
+ source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
+ video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
+ kMaxPayloadLength);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ EXPECT_EQ(
+ video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
+ adapted_framerate);
+
+ // Trigger underuse, max framerate should go back to codec configured fps.
+ stats = stats_proxy_->GetStats();
+ stats.input_frame_rate = adapted_framerate;
+ stats_proxy_->SetMockStats(stats);
+ video_stream_encoder_->TriggerCpuUnderuse();
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(
+ video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
+ kHighFramerate);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ OveruseDetectorUpdatedOnDegradationPreferenceChange) {
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+ const int kFramerate = 24;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+
+ // Trigger initial configuration.
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
+ video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
+ video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
+ source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
+ video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
+ kMaxPayloadLength);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ EXPECT_EQ(
+ video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
+ kFramerate);
+
+ // Trigger overuse, max framerate should be reduced.
+ VideoSendStream::Stats stats = stats_proxy_->GetStats();
+ stats.input_frame_rate = kFramerate;
+ stats_proxy_->SetMockStats(stats);
+ video_stream_encoder_->TriggerCpuOveruse();
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ int adapted_framerate =
+ video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
+ EXPECT_LT(adapted_framerate, kFramerate);
+
+ // Change degradation preference to not enable framerate scaling. Target
+ // framerate should be changed to codec defined limit.
+ video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_EQ(
+ video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
+ kFramerate);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
+ const int kTooLowBitrateForFrameSizeBps = 10000;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
+ DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
+ DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
+ const int kWidth = 640;
+ const int kHeight = 360;
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+
+ // Expect to drop this frame, the wait should time out.
+ ExpectDroppedFrame();
+
+ // Expect the sink_wants to specify a scaled frame.
+ EXPECT_TRUE_WAIT(
+ video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
+
+ int last_pixel_count = video_source_.sink_wants().max_pixel_count;
+
+ // Next frame is scaled.
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
+
+ // Expect to drop this frame, the wait should time out.
+ ExpectDroppedFrame();
+
+ EXPECT_TRUE_WAIT(
+ video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
+ const int kTooLowBitrateForFrameSizeBps = 10000;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
+ DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
+ DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
+ const int kWidth = 640;
+ const int kHeight = 360;
+
+ // We expect the n initial frames to get dropped.
+ int i;
+ for (i = 1; i <= kMaxInitialFramedrop; ++i) {
+ video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
+ ExpectDroppedFrame();
+ }
+ // The n+1th frame should not be dropped, even though it's size is too large.
+ video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
+ WaitForEncodedFrame(i);
+
+ // Expect the sink_wants to specify a scaled frame.
+ EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ InitialFrameDropOffWithMaintainResolutionPreference) {
+ const int kWidth = 640;
+ const int kHeight = 360;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
+
+ // Set degradation preference.
+ video_stream_encoder_->SetSource(
+ &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ // Frame should not be dropped, even if it's too large.
+ WaitForEncodedFrame(1);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
+ const int kWidth = 640;
+ const int kHeight = 360;
+ fake_encoder_.SetQualityScaling(false);
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
+ // Make format different, to force recreation of encoder.
+ video_encoder_config.video_format.parameters["foo"] = "foo";
+ video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
+ kMaxPayloadLength);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
+
+ // Force quality scaler reconfiguration by resetting the source.
+ video_stream_encoder_->SetSource(&video_source_,
+ webrtc::DegradationPreference::BALANCED);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ // Frame should not be dropped, even if it's too large.
+ WaitForEncodedFrame(1);
+
+ video_stream_encoder_->Stop();
+ fake_encoder_.SetQualityScaling(true);
+}
+
+TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
+ webrtc::test::ScopedKeyValueConfig field_trials(
+ field_trials_,
+ "WebRTC-Video-QualityScalerSettings/"
+ "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
+ // Reset encoder for field trials to take effect.
+ ConfigureEncoder(video_encoder_config_.Copy());
+ const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
+ const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
+ const int kWidth = 640;
+ const int kHeight = 360;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ // Frame should not be dropped.
+ WaitForEncodedFrame(1);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
+ DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
+ DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+ // Frame should not be dropped.
+ WaitForEncodedFrame(2);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
+ DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
+ DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+ // Expect to drop this frame, the wait should time out.
+ ExpectDroppedFrame();
+
+ // Expect the sink_wants to specify a scaled frame.
+ EXPECT_TRUE_WAIT(
+ video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
+ webrtc::test::ScopedKeyValueConfig field_trials(
+ field_trials_,
+ "WebRTC-Video-QualityScalerSettings/"
+ "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
+ fake_encoder_.SetQualityScaling(false);
+ ConfigureEncoder(video_encoder_config_.Copy());
+ const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
+ const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
+ const int kWidth = 640;
+ const int kHeight = 360;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ // Frame should not be dropped.
+ WaitForEncodedFrame(1);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
+ DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
+ DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+ // Frame should not be dropped.
+ WaitForEncodedFrame(2);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
+ DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
+ DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+ // Not dropped since quality scaling is disabled.
+ WaitForEncodedFrame(3);
+
+ // Expect the sink_wants to specify a scaled frame.
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
+ const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
+ // Set simulcast.
+ ResetEncoder("VP8", 3, 1, 1, false);
+ fake_encoder_.SetQualityScaling(true);
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ // Frame should not be dropped.
+ WaitForEncodedFrame(1);
+
+ // Trigger QVGA "singlecast"
+ // Update the config.
+ VideoEncoderConfig video_encoder_config;
+ webrtc::VideoEncoder::EncoderInfo encoder_info;
+ test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
+ &video_encoder_config);
+ video_encoder_config.video_stream_factory =
+ rtc::make_ref_counted<cricket::EncoderStreamFactory>(
+ "VP8", /*max qp*/ 56, /*screencast*/ false,
+ /*screenshare enabled*/ false, encoder_info);
+ for (auto& layer : video_encoder_config.simulcast_layers) {
+ layer.num_temporal_layers = 1;
+ layer.max_framerate = kDefaultFramerate;
+ }
+ video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+
+ video_encoder_config.simulcast_layers[0].active = true;
+ video_encoder_config.simulcast_layers[1].active = false;
+ video_encoder_config.simulcast_layers[2].active = false;
+
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+ // Frame should not be dropped.
+ WaitForEncodedFrame(2);
+
+ // Trigger HD "singlecast"
+ video_encoder_config.simulcast_layers[0].active = false;
+ video_encoder_config.simulcast_layers[1].active = false;
+ video_encoder_config.simulcast_layers[2].active = true;
+
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+ // Frame should be dropped because of initial frame drop.
+ ExpectDroppedFrame();
+
+ // Expect the sink_wants to specify a scaled frame.
+ EXPECT_TRUE_WAIT(
+ video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
+ const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
+ // Set simulcast.
+ ResetEncoder("VP9", 1, 1, 3, false);
+ fake_encoder_.SetQualityScaling(true);
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ // Frame should not be dropped.
+ WaitForEncodedFrame(1);
+
+ // Trigger QVGA "singlecast"
+ // Update the config.
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
+ &video_encoder_config);
+ VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
+ vp9_settings.numberOfSpatialLayers = 3;
+ // Since only one layer is active - automatic resize should be enabled.
+ vp9_settings.automaticResizeOn = true;
+ video_encoder_config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
+ vp9_settings);
+ video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+ // Currently simulcast layers `active` flags are used to inidicate
+ // which SVC layers are active.
+ video_encoder_config.simulcast_layers.resize(3);
+
+ video_encoder_config.simulcast_layers[0].active = true;
+ video_encoder_config.simulcast_layers[1].active = false;
+ video_encoder_config.simulcast_layers[2].active = false;
+
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+ // Frame should not be dropped.
+ WaitForEncodedFrame(2);
+
+ // Trigger HD "singlecast"
+ video_encoder_config.simulcast_layers[0].active = false;
+ video_encoder_config.simulcast_layers[1].active = false;
+ video_encoder_config.simulcast_layers[2].active = true;
+
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+ // Frame should be dropped because of initial frame drop.
+ ExpectDroppedFrame();
+
+ // Expect the sink_wants to specify a scaled frame.
+ EXPECT_TRUE_WAIT(
+ video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
+ const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
+ 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
+ const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
+ 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
+ const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
+ 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
+ fake_encoder_.SetResolutionBitrateLimits(
+ {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
+ &video_encoder_config);
+ VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
+ vp9_settings.numberOfSpatialLayers = 3;
+ // Since only one layer is active - automatic resize should be enabled.
+ vp9_settings.automaticResizeOn = true;
+ video_encoder_config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
+ vp9_settings);
+ video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+ // Simulcast layers are used to indicate which spatial layers are active.
+ video_encoder_config.simulcast_layers.resize(3);
+ video_encoder_config.simulcast_layers[0].active = false;
+ video_encoder_config.simulcast_layers[1].active = true;
+ video_encoder_config.simulcast_layers[2].active = false;
+
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+
+ // The encoder bitrate limits for 360p should be used.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
+ EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
+ EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
+ EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
+ EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
+ EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
+ fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
+ fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
+
+ // The encoder bitrate limits for 270p should be used.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
+ EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
+ EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
+ EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
+ EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
+ EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
+ fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
+ fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
+ &video_encoder_config);
+ VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
+ vp9_settings.numberOfSpatialLayers = 3;
+ // Since only one layer is active - automatic resize should be enabled.
+ vp9_settings.automaticResizeOn = true;
+ video_encoder_config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
+ vp9_settings);
+ video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+ // Simulcast layers are used to indicate which spatial layers are active.
+ video_encoder_config.simulcast_layers.resize(3);
+ video_encoder_config.simulcast_layers[0].active = false;
+ video_encoder_config.simulcast_layers[1].active = true;
+ video_encoder_config.simulcast_layers[2].active = false;
+
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+
+ // The default bitrate limits for 360p should be used.
+ const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
+ EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
+ kVideoCodecVP9, 640 * 360);
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
+ EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
+ EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
+ EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
+ EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
+ EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
+ EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
+ fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
+ fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
+
+ // The default bitrate limits for 270p should be used.
+ const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
+ EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
+ kVideoCodecVP9, 480 * 270);
+ video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
+ EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
+ EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
+ EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
+ EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
+ EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
+ EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
+ fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
+ EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
+ fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
+ webrtc::test::ScopedKeyValueConfig field_trials(
+ field_trials_, "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
+ &video_encoder_config);
+ VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
+ vp9_settings.numberOfSpatialLayers = 3;
+ // Since only one layer is active - automatic resize should be enabled.
+ vp9_settings.automaticResizeOn = true;
+ video_encoder_config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
+ vp9_settings);
+ video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+ // Simulcast layers are used to indicate which spatial layers are active.
+ video_encoder_config.simulcast_layers.resize(3);
+ video_encoder_config.simulcast_layers[0].active = false;
+ video_encoder_config.simulcast_layers[1].active = true;
+ video_encoder_config.simulcast_layers[2].active = false;
+
+ // Reset encoder for field trials to take effect.
+ ConfigureEncoder(video_encoder_config.Copy());
+
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+
+ // The default bitrate limits for 360p should not be used.
+ const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
+ EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
+ kVideoCodecVP9, 640 * 360);
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
+ EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
+ EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
+ EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
+ EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
+ EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
+ EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
+ fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
+ ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
+ /*num_spatial_layers=*/1, /*screenshare=*/false);
+
+ // The default singlecast bitrate limits for 720p should not be used.
+ const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
+ EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
+ kVideoCodecVP9, 1280 * 720);
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
+ EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
+ EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
+ EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
+ EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
+ EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
+ EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
+ fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
+ const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
+ 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
+ const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
+ 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
+ fake_encoder_.SetResolutionBitrateLimits(
+ {kEncoderLimits180p, kEncoderLimits720p});
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
+ &video_encoder_config);
+ VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
+ vp9_settings.numberOfSpatialLayers = 3;
+ // Since only one layer is active - automatic resize should be enabled.
+ vp9_settings.automaticResizeOn = true;
+ video_encoder_config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
+ vp9_settings);
+ video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+ // Simulcast layers are used to indicate which spatial layers are active.
+ video_encoder_config.simulcast_layers.resize(3);
+ video_encoder_config.simulcast_layers[0].active = true;
+ video_encoder_config.simulcast_layers[1].active = false;
+ video_encoder_config.simulcast_layers[2].active = false;
+
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+
+ // Limits not applied on lowest stream, limits for 180p should not be used.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
+ EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
+ EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
+ EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
+ EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
+ EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
+ EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
+ fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
+ EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
+ fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ InitialFrameDropActivatesWhenResolutionIncreases) {
+ const int kWidth = 640;
+ const int kHeight = 360;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
+ // Frame should not be dropped.
+ WaitForEncodedFrame(1);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
+ // Frame should not be dropped, bitrate not too low for frame.
+ WaitForEncodedFrame(2);
+
+ // Incoming resolution increases.
+ video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+ // Expect to drop this frame, bitrate too low for frame.
+ ExpectDroppedFrame();
+
+ // Expect the sink_wants to specify a scaled frame.
+ EXPECT_TRUE_WAIT(
+ video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
+ const int kWidth = 640;
+ const int kHeight = 360;
+ // So that quality scaling doesn't happen by itself.
+ fake_encoder_.SetQp(kQpHigh);
+
+ AdaptingFrameForwarder source(&time_controller_);
+ source.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+
+ int timestamp = 1;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp);
+ timestamp += 9000;
+ // Long pause to disable all first BWE drop logic.
+ AdvanceTime(TimeDelta::Millis(1000));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
+ source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
+ // Not dropped frame, as initial frame drop is disabled by now.
+ WaitForEncodedFrame(timestamp);
+ timestamp += 9000;
+ AdvanceTime(TimeDelta::Millis(100));
+
+ // Quality adaptation down.
+ video_stream_encoder_->TriggerQualityLow();
+
+ // Adaptation has an effect.
+ EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
+ 5000);
+
+ // Frame isn't dropped as initial frame dropper is disabled.
+ source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp);
+ timestamp += 9000;
+ AdvanceTime(TimeDelta::Millis(100));
+
+ // Quality adaptation up.
+ video_stream_encoder_->TriggerQualityHigh();
+
+ // Adaptation has an effect.
+ EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
+ 5000);
+
+ source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
+ // Frame should not be dropped, as initial framedropper is off.
+ WaitForEncodedFrame(timestamp);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
+ const int kMinStartBps360p = 222000;
+ fake_encoder_.SetResolutionBitrateLimits(
+ {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
+ VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
+ 800000)});
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
+ DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
+ DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
+ 0, 0, 0);
+ // Frame should not be dropped, bitrate not too low for frame.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
+ WaitForEncodedFrame(1);
+
+ // Incoming resolution increases, initial frame drop activates.
+ // Frame should be dropped, link allocation too low for frame.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
+ ExpectDroppedFrame();
+
+ // Expect sink_wants to specify a scaled frame.
+ EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
+ 5000);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
+ const int kMinStartBps360p = 222000;
+ fake_encoder_.SetResolutionBitrateLimits(
+ {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
+ VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
+ 800000)});
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
+ DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
+ DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
+ 0, 0, 0);
+ // Frame should not be dropped, bitrate not too low for frame.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
+ WaitForEncodedFrame(1);
+
+ // Incoming resolution increases, initial frame drop activates.
+ // Frame should be dropped, link allocation not too low for frame.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
+ WaitForEncodedFrame(2);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
+ webrtc::test::ScopedKeyValueConfig field_trials(
+ field_trials_,
+ "WebRTC-Video-QualityRampupSettings/"
+ "min_pixels:921600,min_duration_ms:2000/");
+
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ const int kFps = 10;
+ max_framerate_ = kFps;
+
+ // Reset encoder for field trials to take effect.
+ VideoEncoderConfig config = video_encoder_config_.Copy();
+ config.max_bitrate_bps = kTargetBitrate.bps();
+ DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
+ ConfigureEncoder(std::move(config));
+ fake_encoder_.SetQp(kQpLow);
+
+ // Enable MAINTAIN_FRAMERATE preference.
+ AdaptingFrameForwarder source(&time_controller_);
+ source.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(&source,
+ DegradationPreference::MAINTAIN_FRAMERATE);
+
+ // Start at low bitrate.
+ const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
+
+ // Expect first frame to be dropped and resolution to be limited.
+ const int64_t kFrameIntervalMs = 1000 / kFps;
+ int64_t timestamp_ms = kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ ExpectDroppedFrame();
+ EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
+ 5000);
+
+ // Increase bitrate to encoder max.
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
+
+ // Insert frames and advance `min_duration_ms`.
+ const int64_t start_bw_high_ms = CurrentTimeMs();
+ for (size_t i = 1; i <= 10; i++) {
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ }
+
+ // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
+ int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
+ AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
+
+ // Frame should trigger high BW and release quality limitation.
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ // The ramp-up code involves the adaptation queue, give it time to execute.
+ // TODO(hbos): Can we await an appropriate event instead?
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+
+ // Frame should not be adapted.
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
+ webrtc::test::ScopedKeyValueConfig field_trials(
+ field_trials_, "WebRTC-Video-QualityScaling/Disabled/");
+ AdaptingFrameForwarder source(&time_controller_);
+ source.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(&source,
+ DegradationPreference::MAINTAIN_FRAMERATE);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ fake_encoder_.SetQp(kQpHigh + 1);
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ const int64_t kFrameIntervalMs = 100;
+ int64_t timestamp_ms = kFrameIntervalMs;
+ for (size_t i = 1; i <= 100; i++) {
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ }
+ // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
+ // for the first time.
+ // TODO(eshr): We should avoid these waits by using threads with simulated
+ // time.
+ EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
+ 2000 * 2.5 * 2);
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ // Disable Quality scaling by turning off scaler on the encoder and
+ // reconfiguring.
+ fake_encoder_.SetQualityScaling(false);
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
+ kMaxPayloadLength);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ AdvanceTime(TimeDelta::Zero());
+ // Since we turned off the quality scaler, the adaptations made by it are
+ // removed.
+ EXPECT_THAT(source.sink_wants(), ResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
+ const int kTooSmallWidth = 10;
+ const int kTooSmallHeight = 10;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+
+ // Trigger adapt down, too small frame, expect no change.
+ source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerCpuOveruse();
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
+ const int kTooSmallWidth = 10;
+ const int kTooSmallHeight = 10;
+ const int kFpsLimit = 7;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable BALANCED preference, no initial limitation.
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(&source,
+ webrtc::DegradationPreference::BALANCED);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+
+ // Trigger adapt down, expect limited framerate.
+ source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, too small frame, expect no change.
+ source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
+ WaitForEncodedFrame(2);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
+ fake_encoder_.ForceInitEncodeFailure(true);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ ResetEncoder("VP8", 2, 1, 1, false);
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(1, kFrameWidth, kFrameHeight));
+ ExpectDroppedFrame();
+ video_stream_encoder_->Stop();
+}
+
+// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
+TEST_F(VideoStreamEncoderTest,
+ AdaptsResolutionOnOveruse_MaintainFramerateMode) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+ // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
+ // requested by
+ // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
+ video_source_.set_adaptation_enabled(true);
+
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(kFrameWidth, kFrameHeight);
+
+ // Trigger CPU overuse, downscale by 3/4.
+ video_stream_encoder_->TriggerCpuOveruse();
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
+
+ // Trigger CPU normal use, return to original resolution.
+ video_stream_encoder_->TriggerCpuUnderuse();
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(kFrameWidth, kFrameHeight);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ AdaptsFramerateOnOveruse_MaintainResolutionMode) {
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_stream_encoder_->SetSource(
+ &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+ video_source_.set_adaptation_enabled(true);
+
+ int64_t timestamp_ms = CurrentTimeMs();
+
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(timestamp_ms);
+
+ // Try to trigger overuse. No fps estimate available => no effect.
+ video_stream_encoder_->TriggerCpuOveruse();
+
+ // Insert frames for one second to get a stable estimate.
+ for (int i = 0; i < max_framerate_; ++i) {
+ timestamp_ms += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ }
+
+ // Trigger CPU overuse, reduce framerate by 2/3.
+ video_stream_encoder_->TriggerCpuOveruse();
+ int num_frames_dropped = 0;
+ for (int i = 0; i < max_framerate_; ++i) {
+ timestamp_ms += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ if (!WaitForFrame(kFrameTimeout)) {
+ ++num_frames_dropped;
+ } else {
+ sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
+ }
+ }
+
+ // Add some slack to account for frames dropped by the frame dropper.
+ const int kErrorMargin = 1;
+ EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
+ kErrorMargin);
+
+ // Trigger CPU overuse, reduce framerate by 2/3 again.
+ video_stream_encoder_->TriggerCpuOveruse();
+ num_frames_dropped = 0;
+ for (int i = 0; i <= max_framerate_; ++i) {
+ timestamp_ms += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ if (!WaitForFrame(kFrameTimeout)) {
+ ++num_frames_dropped;
+ } else {
+ sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
+ }
+ }
+ EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
+ kErrorMargin);
+
+ // Go back up one step.
+ video_stream_encoder_->TriggerCpuUnderuse();
+ num_frames_dropped = 0;
+ for (int i = 0; i < max_framerate_; ++i) {
+ timestamp_ms += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ if (!WaitForFrame(kFrameTimeout)) {
+ ++num_frames_dropped;
+ } else {
+ sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
+ }
+ }
+ EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
+ kErrorMargin);
+
+ // Go back up to original mode.
+ video_stream_encoder_->TriggerCpuUnderuse();
+ num_frames_dropped = 0;
+ for (int i = 0; i < max_framerate_; ++i) {
+ timestamp_ms += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ if (!WaitForFrame(kFrameTimeout)) {
+ ++num_frames_dropped;
+ } else {
+ sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
+ }
+ }
+ EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
+ const int kFramerateFps = 5;
+ const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+
+ // Reconfigure encoder with two temporal layers and screensharing, which will
+ // disable frame dropping and make testing easier.
+ ResetEncoder("VP8", 1, 2, 1, true);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_stream_encoder_->SetSource(
+ &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
+ video_source_.set_adaptation_enabled(true);
+
+ int64_t timestamp_ms = CurrentTimeMs();
+
+ // Trigger overuse as much as we can.
+ rtc::VideoSinkWants last_wants;
+ do {
+ last_wants = video_source_.sink_wants();
+
+ // Insert frames to get a new fps estimate...
+ for (int j = 0; j < kFramerateFps; ++j) {
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ if (video_source_.last_sent_width()) {
+ sink_.WaitForEncodedFrame(timestamp_ms);
+ }
+ timestamp_ms += kFrameIntervalMs;
+ AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
+ }
+ // ...and then try to adapt again.
+ video_stream_encoder_->TriggerCpuOveruse();
+ } while (video_source_.sink_wants().max_framerate_fps <
+ last_wants.max_framerate_fps);
+
+ EXPECT_THAT(video_source_.sink_wants(),
+ FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ const int64_t kFrameIntervalMs = 150;
+ int64_t timestamp_ms = kFrameIntervalMs;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable BALANCED preference, no initial limitation.
+ AdaptingFrameForwarder source(&time_controller_);
+ source.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(&source,
+ webrtc::DegradationPreference::BALANCED);
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down resolution (960x540@30fps).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(),
+ FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down resolution (640x360@30fps).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect reduced fps (640x360@15fps).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down resolution (480x270@15fps).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect scaled down resolution (320x180@10fps).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, expect reduced fps (320x180@7fps).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
+ rtc::VideoSinkWants last_wants = source.sink_wants();
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt down, min resolution reached, expect no change.
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect increased fps (320x180@10fps).
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect upscaled resolution (480x270@10fps).
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect upscaled resolution (640x360@15fps).
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect increased fps (640x360@30fps).
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsMax());
+ EXPECT_EQ(source.sink_wants().max_pixel_count,
+ source.last_wants().max_pixel_count);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect upscaled resolution (960x540@30fps).
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect no restriction (1280x720fps@30fps).
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect no change.
+ video_stream_encoder_->TriggerQualityHigh();
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ const int64_t kFrameIntervalMs = 150;
+ int64_t timestamp_ms = kFrameIntervalMs;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable BALANCED preference, no initial limitation.
+ AdaptingFrameForwarder source(&time_controller_);
+ source.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(&source,
+ webrtc::DegradationPreference::BALANCED);
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
+ video_stream_encoder_->TriggerCpuOveruse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(),
+ FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
+ video_stream_encoder_->TriggerCpuOveruse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger quality adapt down, expect reduced fps (640x360@15fps).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger cpu adapt up, expect no change since QP is most limited.
+ {
+ // Store current sink wants since we expect no change and if there is no
+ // change then last_wants() is not updated.
+ auto previous_sink_wants = source.sink_wants();
+ video_stream_encoder_->TriggerCpuUnderuse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+ }
+
+ // Trigger quality adapt up, expect increased fps (640x360@30fps).
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger quality adapt up and Cpu adapt up since both are most limited,
+ // expect increased resolution (960x540@30fps).
+ video_stream_encoder_->TriggerQualityHigh();
+ video_stream_encoder_->TriggerCpuUnderuse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger quality adapt up and Cpu adapt up since both are most limited,
+ // expect no restriction (1280x720fps@30fps).
+ video_stream_encoder_->TriggerQualityHigh();
+ video_stream_encoder_->TriggerCpuUnderuse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect no change.
+ video_stream_encoder_->TriggerQualityHigh();
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
+ const int kWidth = 640;
+ const int kHeight = 360;
+ const int kFpsLimit = 15;
+ const int64_t kFrameIntervalMs = 150;
+ int64_t timestamp_ms = kFrameIntervalMs;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Enable BALANCED preference, no initial limitation.
+ AdaptingFrameForwarder source(&time_controller_);
+ source.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(&source,
+ webrtc::DegradationPreference::BALANCED);
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(kWidth, kHeight);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
+ video_stream_encoder_->TriggerCpuOveruse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
+ video_stream_encoder_->TriggerQualityLow();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger cpu adapt up, expect no change because quality is most limited.
+ {
+ auto previous_sink_wants = source.sink_wants();
+ // Store current sink wants since we expect no change ind if there is no
+ // change then last__wants() is not updated.
+ video_stream_encoder_->TriggerCpuUnderuse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+ }
+
+ // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
+ video_stream_encoder_->TriggerQualityHigh();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
+ video_stream_encoder_->TriggerQualityHigh();
+ video_stream_encoder_->TriggerCpuUnderuse();
+ timestamp_ms += kFrameIntervalMs;
+ source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+ EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ // Trigger adapt up, expect no change.
+ video_stream_encoder_->TriggerQualityHigh();
+ EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
+ EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+ EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
+ const int kFrameWidth = 1920;
+ const int kFrameHeight = 1080;
+ // 2/3 of 1920.
+ const int kAdaptedFrameWidth = 1280;
+ // 2/3 of 1080.
+ const int kAdaptedFrameHeight = 720;
+ const int kFramerate = 24;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ // Trigger reconfigure encoder (without resetting the entire instance).
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
+ video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
+ video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
+ video_encoder_config.video_stream_factory =
+ rtc::make_ref_counted<CroppingVideoStreamFactory>();
+ video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
+ kMaxPayloadLength);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ video_source_.set_adaptation_enabled(true);
+
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(1, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(kFrameWidth, kFrameHeight);
+
+ // Trigger CPU overuse, downscale by 3/4.
+ video_stream_encoder_->TriggerCpuOveruse();
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(2, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+ const int kLowFps = 2;
+ const int kHighFps = 30;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ int64_t timestamp_ms = CurrentTimeMs();
+ max_framerate_ = kLowFps;
+
+ // Insert 2 seconds of 2fps video.
+ for (int i = 0; i < kLowFps * 2; ++i) {
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ timestamp_ms += 1000 / kLowFps;
+ }
+
+ // Make sure encoder is updated with new target.
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ timestamp_ms += 1000 / kLowFps;
+
+ EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
+
+ // Insert 30fps frames for just a little more than the forced update period.
+ const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
+ constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / kHighFps;
+ max_framerate_ = kHighFps;
+ for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
+ // be dropped if the encoder hans't been updated with the new higher target
+ // framerate yet, causing it to overshoot the target bitrate and then
+ // suffering the wrath of the media optimizer.
+ TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameInterval);
+ timestamp_ms += kFrameInterval.ms();
+ }
+
+ // Don expect correct measurement just yet, but it should be higher than
+ // before.
+ EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+ ResetEncoder("FAKE", 1, 1, 1, false,
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoBitrateAllocation);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ // Insert a first video frame, causes another bitrate update.
+ int64_t timestamp_ms = CurrentTimeMs();
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
+
+ // Next, simulate video suspension due to pacer queue overrun.
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
+
+ // Skip ahead until a new periodic parameter update should have occured.
+ timestamp_ms += kProcessIntervalMs;
+ AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
+
+ // No more allocations has been made.
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ ExpectDroppedFrame();
+ EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+ const test::ScopedKeyValueConfig kFieldTrials;
+ const CpuOveruseOptions default_options(kFieldTrials);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(1, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
+ .low_encode_usage_threshold_percent,
+ default_options.low_encode_usage_threshold_percent);
+ EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
+ .high_encode_usage_threshold_percent,
+ default_options.high_encode_usage_threshold_percent);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ HigherCpuAdaptationThresholdsForHardwareEncoder) {
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+ const test::ScopedKeyValueConfig kFieldTrials;
+ CpuOveruseOptions hardware_options(kFieldTrials);
+ hardware_options.low_encode_usage_threshold_percent = 150;
+ hardware_options.high_encode_usage_threshold_percent = 200;
+ fake_encoder_.SetIsHardwareAccelerated(true);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(1, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
+ .low_encode_usage_threshold_percent,
+ hardware_options.low_encode_usage_threshold_percent);
+ EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
+ .high_encode_usage_threshold_percent,
+ hardware_options.high_encode_usage_threshold_percent);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+
+ const test::ScopedKeyValueConfig kFieldTrials;
+ const CpuOveruseOptions default_options(kFieldTrials);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(1, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
+ .low_encode_usage_threshold_percent,
+ default_options.low_encode_usage_threshold_percent);
+ EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
+ .high_encode_usage_threshold_percent,
+ default_options.high_encode_usage_threshold_percent);
+
+ CpuOveruseOptions hardware_options(kFieldTrials);
+ hardware_options.low_encode_usage_threshold_percent = 150;
+ hardware_options.high_encode_usage_threshold_percent = 200;
+ fake_encoder_.SetIsHardwareAccelerated(true);
+
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(2, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(2);
+
+ EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
+ .low_encode_usage_threshold_percent,
+ hardware_options.low_encode_usage_threshold_percent);
+ EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
+ .high_encode_usage_threshold_percent,
+ hardware_options.high_encode_usage_threshold_percent);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
+ const int kFrameWidth = 320;
+ const int kFrameHeight = 240;
+ const int kFps = 30;
+ const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
+ const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ int64_t timestamp_ms = CurrentTimeMs();
+ max_framerate_ = kFps;
+
+ // Insert 3 seconds of video, verify number of drops with normal bitrate.
+ fake_encoder_.SimulateOvershoot(1.0);
+ int num_dropped = 0;
+ for (int i = 0; i < kNumFramesInRun; ++i) {
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ // Wait up to two frame durations for a frame to arrive.
+ if (!TimedWaitForEncodedFrame(timestamp_ms,
+ 2 * TimeDelta::Seconds(1) / kFps)) {
+ ++num_dropped;
+ }
+ timestamp_ms += 1000 / kFps;
+ }
+
+ // Framerate should be measured to be near the expected target rate.
+ EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
+
+ // Frame drops should be within 5% of expected 0%.
+ EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
+
+ // Make encoder produce frames at double the expected bitrate during 3 seconds
+ // of video, verify number of drops. Rate needs to be slightly changed in
+ // order to force the rate to be reconfigured.
+ double overshoot_factor = 2.0;
+ const RateControlSettings trials =
+ RateControlSettings::ParseFromFieldTrials();
+ if (trials.UseEncoderBitrateAdjuster()) {
+ // With bitrate adjuster, when need to overshoot even more to trigger
+ // frame dropping since the adjuter will try to just lower the target
+ // bitrate rather than drop frames. If network headroom can be used, it
+ // doesn't push back as hard so we don't need quite as much overshoot.
+ // These numbers are unfortunately a bit magical but there's not trivial
+ // way to algebraically infer them.
+ overshoot_factor = 3.0;
+ }
+ fake_encoder_.SimulateOvershoot(overshoot_factor);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate + DataRate::KilobitsPerSec(1),
+ kTargetBitrate + DataRate::KilobitsPerSec(1),
+ kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
+ num_dropped = 0;
+ for (int i = 0; i < kNumFramesInRun; ++i) {
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ // Wait up to two frame durations for a frame to arrive.
+ if (!TimedWaitForEncodedFrame(timestamp_ms,
+ 2 * TimeDelta::Seconds(1) / kFps)) {
+ ++num_dropped;
+ }
+ timestamp_ms += 1000 / kFps;
+ }
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Target framerate should be still be near the expected target, despite
+ // the frame drops.
+ EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
+
+ // Frame drops should be within 5% of expected 50%.
+ EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
+ const int kFrameWidth = 320;
+ const int kFrameHeight = 240;
+ const int kActualInputFps = 24;
+ const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
+
+ ASSERT_GT(max_framerate_, kActualInputFps);
+
+ int64_t timestamp_ms = CurrentTimeMs();
+ max_framerate_ = kActualInputFps;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Insert 3 seconds of video, with an input fps lower than configured max.
+ for (int i = 0; i < kActualInputFps * 3; ++i) {
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ // Wait up to two frame durations for a frame to arrive.
+ WaitForEncodedFrame(timestamp_ms);
+ timestamp_ms += 1000 / kActualInputFps;
+ }
+
+ EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
+ VideoFrame::UpdateRect rect;
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(&source,
+ DegradationPreference::MAINTAIN_FRAMERATE);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
+ WaitForEncodedFrame(1);
+ // On the very first frame full update should be forced.
+ rect = fake_encoder_.GetLastUpdateRect();
+ EXPECT_EQ(rect.offset_x, 0);
+ EXPECT_EQ(rect.offset_y, 0);
+ EXPECT_EQ(rect.height, codec_height_);
+ EXPECT_EQ(rect.width, codec_width_);
+ // Frame with NTP timestamp 2 will be dropped due to outstanding frames
+ // scheduled for processing during encoder queue processing of frame 2.
+ source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
+ source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
+ WaitForEncodedFrame(3);
+ // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
+ rect = fake_encoder_.GetLastUpdateRect();
+ EXPECT_EQ(rect.offset_x, 1);
+ EXPECT_EQ(rect.offset_y, 0);
+ EXPECT_EQ(rect.width, 10);
+ EXPECT_EQ(rect.height, 1);
+
+ source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
+ WaitForEncodedFrame(4);
+ // Previous frame was encoded, so no accumulation should happen.
+ rect = fake_encoder_.GetLastUpdateRect();
+ EXPECT_EQ(rect.offset_x, 0);
+ EXPECT_EQ(rect.offset_y, 0);
+ EXPECT_EQ(rect.width, 1);
+ EXPECT_EQ(rect.height, 1);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // First frame is always keyframe.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ EXPECT_THAT(
+ fake_encoder_.LastFrameTypes(),
+ ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
+
+ // Insert delta frame.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
+ WaitForEncodedFrame(2);
+ EXPECT_THAT(
+ fake_encoder_.LastFrameTypes(),
+ ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
+
+ // Request next frame be a key-frame.
+ video_stream_encoder_->SendKeyFrame();
+ video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
+ WaitForEncodedFrame(3);
+ EXPECT_THAT(
+ fake_encoder_.LastFrameTypes(),
+ ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
+ // Setup simulcast with three streams.
+ ResetEncoder("VP8", 3, 1, 1, false);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
+ 0, 0, 0);
+ // Wait for all three layers before triggering event.
+ sink_.SetNumExpectedLayers(3);
+
+ // First frame is always keyframe.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
+ VideoFrameType::kVideoFrameKey,
+ VideoFrameType::kVideoFrameKey}));
+
+ // Insert delta frame.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
+ WaitForEncodedFrame(2);
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
+ VideoFrameType::kVideoFrameDelta,
+ VideoFrameType::kVideoFrameDelta}));
+
+ // Request next frame be a key-frame.
+ // Only first stream is configured to produce key-frame.
+ video_stream_encoder_->SendKeyFrame();
+ video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
+ WaitForEncodedFrame(3);
+
+ // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
+ // keyframe request on any layer triggers keyframe on all layers.
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
+ VideoFrameType::kVideoFrameKey,
+ VideoFrameType::kVideoFrameKey}));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
+ // SPS contains VUI with restrictions on the maximum number of reordered
+ // pictures, there is no need to rewrite the bitstream to enable faster
+ // decoding.
+ ResetEncoder("H264", 1, 1, 1, false);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ fake_encoder_.SetEncodedImageData(
+ EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+
+ EXPECT_THAT(sink_.GetLastEncodedImageData(),
+ testing::ElementsAreArray(kOptimalSps));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
+ // SPS does not contain VUI, the bitstream is will be rewritten with added
+ // VUI with restrictions on the maximum number of reordered pictures to
+ // enable faster decoding.
+ uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
+ 0x00, 0x00, 0x03, 0x03, 0xF4,
+ 0x05, 0x03, 0xC7, 0xC0};
+ ResetEncoder("H264", 1, 1, 1, false);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ fake_encoder_.SetEncodedImageData(
+ EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+
+ EXPECT_THAT(sink_.GetLastEncodedImageData(),
+ testing::ElementsAreArray(kOptimalSps));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+ const DataRate kTargetBitrate =
+ DataRate::KilobitsPerSec(300); // Too low for HD resolution.
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ // Insert a first video frame. It should be dropped because of downscale in
+ // resolution.
+ int64_t timestamp_ms = CurrentTimeMs();
+ VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
+ frame.set_rotation(kVideoRotation_270);
+ video_source_.IncomingCapturedFrame(frame);
+
+ ExpectDroppedFrame();
+
+ // Second frame is downscaled.
+ timestamp_ms = CurrentTimeMs();
+ frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
+ frame.set_rotation(kVideoRotation_90);
+ video_source_.IncomingCapturedFrame(frame);
+
+ WaitForEncodedFrame(timestamp_ms);
+ sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
+
+ // Insert another frame, also downscaled.
+ timestamp_ms = CurrentTimeMs();
+ frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
+ frame.set_rotation(kVideoRotation_180);
+ video_source_.IncomingCapturedFrame(frame);
+
+ WaitForEncodedFrame(timestamp_ms);
+ sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
+ const int kFrameWidth = 320;
+ const int kFrameHeight = 180;
+
+ // Initial rate.
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ /*target_bitrate=*/DataRate::KilobitsPerSec(300),
+ /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
+ /*link_allocation=*/DataRate::KilobitsPerSec(300),
+ /*fraction_lost=*/0,
+ /*round_trip_time_ms=*/0,
+ /*cwnd_reduce_ratio=*/0);
+
+ // Insert a first video frame so that encoder gets configured.
+ int64_t timestamp_ms = CurrentTimeMs();
+ VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
+ frame.set_rotation(kVideoRotation_270);
+ video_source_.IncomingCapturedFrame(frame);
+ WaitForEncodedFrame(timestamp_ms);
+
+ // Set a target rate below the minimum allowed by the codec settings.
+ VideoCodec codec_config = fake_encoder_.config();
+ DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
+ DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ /*target_bitrate=*/target_rate,
+ /*stable_target_bitrate=*/target_rate,
+ /*link_allocation=*/target_rate,
+ /*fraction_lost=*/0,
+ /*round_trip_time_ms=*/0,
+ /*cwnd_reduce_ratio=*/0);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ // Target bitrate and bandwidth allocation should both be capped at min_rate.
+ auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
+ ASSERT_TRUE(rate_settings.has_value());
+ DataRate allocation_sum =
+ DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
+ EXPECT_EQ(min_rate, allocation_sum);
+ EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ int64_t timestamp_ms = CurrentTimeMs();
+ video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
+ WaitForEncodedFrame(1);
+
+ auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
+ ASSERT_TRUE(prev_rate_settings.has_value());
+ EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
+ kDefaultFramerate);
+
+ // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
+ for (int i = 0; i < 2 * kDefaultFramerate; i++) {
+ timestamp_ms += 1000 / kDefaultFramerate;
+ video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
+ WaitForEncodedFrame(timestamp_ms);
+ }
+ EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
+ kDefaultFramerate);
+ // Capture larger frame to trigger a reconfigure.
+ codec_height_ *= 2;
+ codec_width_ *= 2;
+ timestamp_ms += 1000 / kDefaultFramerate;
+ video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
+ WaitForEncodedFrame(timestamp_ms);
+
+ EXPECT_EQ(2, sink_.number_of_reconfigurations());
+ auto current_rate_settings =
+ fake_encoder_.GetAndResetLastRateControlSettings();
+ // Ensure we have actually reconfigured twice
+ // The rate settings should have been set again even though
+ // they haven't changed.
+ ASSERT_TRUE(current_rate_settings.has_value());
+ EXPECT_EQ(prev_rate_settings, current_rate_settings);
+
+ video_stream_encoder_->Stop();
+}
+
+struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
+ MOCK_METHOD(void, RequestEncoderFallback, (), (override));
+ MOCK_METHOD(void,
+ RequestEncoderSwitch,
+ (const webrtc::SdpVideoFormat& format,
+ bool allow_default_fallback),
+ (override));
+};
+
+TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
+ constexpr int kDontCare = 100;
+ StrictMock<MockEncoderSelector> encoder_selector;
+ auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
+ &fake_encoder_, &encoder_selector);
+ video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
+
+ // Reset encoder for new configuration to take effect.
+ ConfigureEncoder(video_encoder_config_.Copy());
+
+ EXPECT_CALL(encoder_selector, OnCurrentEncoder);
+
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(kDontCare, kDontCare, kDontCare));
+ AdvanceTime(TimeDelta::Zero());
+ video_stream_encoder_->Stop();
+
+ // The encoders produced by the VideoEncoderProxyFactory have a pointer back
+ // to it's factory, so in order for the encoder instance in the
+ // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
+ // reset the `video_stream_encoder_` here.
+ video_stream_encoder_.reset();
+}
+
+TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
+ constexpr int kDontCare = 100;
+
+ NiceMock<MockEncoderSelector> encoder_selector;
+ StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
+ video_send_config_.encoder_settings.encoder_switch_request_callback =
+ &switch_callback;
+ auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
+ &fake_encoder_, &encoder_selector);
+ video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
+
+ // Reset encoder for new configuration to take effect.
+ ConfigureEncoder(video_encoder_config_.Copy());
+
+ ON_CALL(encoder_selector, OnAvailableBitrate)
+ .WillByDefault(Return(SdpVideoFormat("AV1")));
+ EXPECT_CALL(switch_callback,
+ RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
+ /*allow_default_fallback=*/false));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ /*target_bitrate=*/DataRate::KilobitsPerSec(50),
+ /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
+ /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
+ /*fraction_lost=*/0,
+ /*round_trip_time_ms=*/0,
+ /*cwnd_reduce_ratio=*/0);
+ AdvanceTime(TimeDelta::Zero());
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, EncoderSelectorResolutionSwitch) {
+ NiceMock<MockEncoderSelector> encoder_selector;
+ StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
+ video_send_config_.encoder_settings.encoder_switch_request_callback =
+ &switch_callback;
+ auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
+ &fake_encoder_, &encoder_selector);
+ video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
+
+ // Reset encoder for new configuration to take effect.
+ ConfigureEncoder(video_encoder_config_.Copy());
+
+ EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(640, 480)))
+ .WillOnce(Return(absl::nullopt));
+ EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(320, 240)))
+ .WillOnce(Return(SdpVideoFormat("AV1")));
+ EXPECT_CALL(switch_callback,
+ RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
+ /*allow_default_fallback=*/false));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ /*target_bitrate=*/DataRate::KilobitsPerSec(800),
+ /*stable_target_bitrate=*/DataRate::KilobitsPerSec(1000),
+ /*link_allocation=*/DataRate::KilobitsPerSec(1000),
+ /*fraction_lost=*/0,
+ /*round_trip_time_ms=*/0,
+ /*cwnd_reduce_ratio=*/0);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 640, 480));
+ video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 480));
+ video_source_.IncomingCapturedFrame(CreateFrame(3, 320, 240));
+
+ AdvanceTime(TimeDelta::Zero());
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
+ constexpr int kSufficientBitrateToNotDrop = 1000;
+ constexpr int kDontCare = 100;
+
+ NiceMock<MockVideoEncoder> video_encoder;
+ NiceMock<MockEncoderSelector> encoder_selector;
+ StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
+ video_send_config_.encoder_settings.encoder_switch_request_callback =
+ &switch_callback;
+ auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
+ &video_encoder, &encoder_selector);
+ video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
+
+ // Reset encoder for new configuration to take effect.
+ ConfigureEncoder(video_encoder_config_.Copy());
+
+ // The VideoStreamEncoder needs some bitrate before it can start encoding,
+ // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
+ // not fail.
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
+ /*stable_target_bitrate=*/
+ DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
+ /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
+ /*fraction_lost=*/0,
+ /*round_trip_time_ms=*/0,
+ /*cwnd_reduce_ratio=*/0);
+
+ ON_CALL(video_encoder, Encode)
+ .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
+ ON_CALL(encoder_selector, OnEncoderBroken)
+ .WillByDefault(Return(SdpVideoFormat("AV2")));
+
+ rtc::Event encode_attempted;
+ EXPECT_CALL(switch_callback,
+ RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
+ /*allow_default_fallback=*/true))
+ .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
+ encode_attempted.Wait(TimeDelta::Seconds(3));
+
+ AdvanceTime(TimeDelta::Zero());
+
+ video_stream_encoder_->Stop();
+
+ // The encoders produced by the VideoEncoderProxyFactory have a pointer back
+ // to it's factory, so in order for the encoder instance in the
+ // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
+ // reset the `video_stream_encoder_` here.
+ video_stream_encoder_.reset();
+}
+
+TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
+ NiceMock<MockVideoEncoder> video_encoder;
+ NiceMock<MockEncoderSelector> encoder_selector;
+ StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
+ video_send_config_.encoder_settings.encoder_switch_request_callback =
+ &switch_callback;
+ auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
+ &video_encoder, &encoder_selector);
+ video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
+
+ // Reset encoder for new configuration to take effect.
+ ConfigureEncoder(video_encoder_config_.Copy());
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
+ /*round_trip_time_ms=*/0,
+ /*cwnd_reduce_ratio=*/0);
+ ASSERT_EQ(0, sink_.number_of_reconfigurations());
+
+ ON_CALL(video_encoder, InitEncode(_, _))
+ .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
+ ON_CALL(encoder_selector, OnEncoderBroken)
+ .WillByDefault(Return(SdpVideoFormat("AV2")));
+
+ rtc::Event encode_attempted;
+ EXPECT_CALL(switch_callback,
+ RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
+ /*allow_default_fallback=*/true))
+ .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ encode_attempted.Wait(TimeDelta::Seconds(3));
+
+ AdvanceTime(TimeDelta::Zero());
+
+ video_stream_encoder_->Stop();
+
+ // The encoders produced by the VideoEncoderProxyFactory have a pointer back
+ // to it's factory, so in order for the encoder instance in the
+ // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
+ // reset the `video_stream_encoder_` here.
+ video_stream_encoder_.reset();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ SwitchEncoderOnInitFailureWithoutEncoderSelector) {
+ NiceMock<MockVideoEncoder> video_encoder;
+ StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
+ video_send_config_.encoder_settings.encoder_switch_request_callback =
+ &switch_callback;
+ auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
+ &video_encoder, /*encoder_selector=*/nullptr);
+ video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
+
+ // Reset encoder for new configuration to take effect.
+ ConfigureEncoder(video_encoder_config_.Copy());
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
+ /*round_trip_time_ms=*/0,
+ /*cwnd_reduce_ratio=*/0);
+ ASSERT_EQ(0, sink_.number_of_reconfigurations());
+
+ ON_CALL(video_encoder, InitEncode(_, _))
+ .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
+
+ rtc::Event encode_attempted;
+ EXPECT_CALL(switch_callback,
+ RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
+ /*allow_default_fallback=*/true))
+ .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ encode_attempted.Wait(TimeDelta::Seconds(3));
+
+ AdvanceTime(TimeDelta::Zero());
+
+ video_stream_encoder_->Stop();
+
+ // The encoders produced by the VideoEncoderProxyFactory have a pointer back
+ // to it's factory, so in order for the encoder instance in the
+ // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
+ // reset the `video_stream_encoder_` here.
+ video_stream_encoder_.reset();
+}
+
+TEST_F(VideoStreamEncoderTest, NullEncoderReturnSwitch) {
+ // As a variant of EncoderSelectorBrokenEncoderSwitch, when a null
+ // VideoEncoder is passed in encoder_factory, it checks whether
+ // Codec Switch occurs without a crash.
+ constexpr int kSufficientBitrateToNotDrop = 1000;
+ constexpr int kDontCare = 100;
+
+ NiceMock<MockEncoderSelector> encoder_selector;
+ StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
+ video_send_config_.encoder_settings.encoder_switch_request_callback =
+ &switch_callback;
+ auto encoder_factory =
+ std::make_unique<test::VideoEncoderNullableProxyFactory>(
+ /*encoder=*/nullptr, &encoder_selector);
+ video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
+
+ // Reset encoder for new configuration to take effect.
+ ConfigureEncoder(video_encoder_config_.Copy());
+ // The VideoStreamEncoder needs some bitrate before it can start encoding,
+ // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
+ // not fail.
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
+ /*stable_target_bitrate=*/
+ DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
+ /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
+ /*fraction_lost=*/0,
+ /*round_trip_time_ms=*/0,
+ /*cwnd_reduce_ratio=*/0);
+ ON_CALL(encoder_selector, OnEncoderBroken)
+ .WillByDefault(Return(SdpVideoFormat("AV2")));
+ rtc::Event encode_attempted;
+ EXPECT_CALL(switch_callback,
+ RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
+ /*allow_default_fallback=*/_))
+ .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
+ encode_attempted.Wait(TimeDelta::Seconds(3));
+
+ AdvanceTime(TimeDelta::Zero());
+
+ video_stream_encoder_->Stop();
+
+ // The encoders produced by the VideoEncoderProxyFactory have a pointer back
+ // to it's factory, so in order for the encoder instance in the
+ // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
+ // reset the `video_stream_encoder_` here.
+ video_stream_encoder_.reset();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ AllocationPropagatedToEncoderWhenTargetRateChanged) {
+ const int kFrameWidth = 320;
+ const int kFrameHeight = 180;
+
+ // Set initial rate.
+ auto rate = DataRate::KilobitsPerSec(100);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ /*target_bitrate=*/rate,
+ /*stable_target_bitrate=*/rate,
+ /*link_allocation=*/rate,
+ /*fraction_lost=*/0,
+ /*round_trip_time_ms=*/0,
+ /*cwnd_reduce_ratio=*/0);
+
+ // Insert a first video frame so that encoder gets configured.
+ int64_t timestamp_ms = CurrentTimeMs();
+ VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
+ frame.set_rotation(kVideoRotation_270);
+ video_source_.IncomingCapturedFrame(frame);
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
+
+ // Change of target bitrate propagates to the encoder.
+ auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ /*target_bitrate=*/new_stable_rate,
+ /*stable_target_bitrate=*/new_stable_rate,
+ /*link_allocation=*/rate,
+ /*fraction_lost=*/0,
+ /*round_trip_time_ms=*/0,
+ /*cwnd_reduce_ratio=*/0);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
+ const int kFrameWidth = 320;
+ const int kFrameHeight = 180;
+
+ // Set initial rate.
+ auto rate = DataRate::KilobitsPerSec(100);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ /*target_bitrate=*/rate,
+ /*stable_target_bitrate=*/rate,
+ /*link_allocation=*/rate,
+ /*fraction_lost=*/0,
+ /*round_trip_time_ms=*/0,
+ /*cwnd_reduce_ratio=*/0);
+
+ // Insert a first video frame so that encoder gets configured.
+ int64_t timestamp_ms = CurrentTimeMs();
+ VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
+ frame.set_rotation(kVideoRotation_270);
+ video_source_.IncomingCapturedFrame(frame);
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
+
+ // Set a higher target rate without changing the link_allocation. Should not
+ // reset encoder's rate.
+ auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ /*target_bitrate=*/rate,
+ /*stable_target_bitrate=*/new_stable_rate,
+ /*link_allocation=*/rate,
+ /*fraction_lost=*/0,
+ /*round_trip_time_ms=*/0,
+ /*cwnd_reduce_ratio=*/0);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
+ test::ScopedKeyValueConfig field_trials(
+ field_trials_,
+ "WebRTC-AutomaticAnimationDetectionScreenshare/"
+ "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
+ const int kFramerateFps = 30;
+ const int kWidth = 1920;
+ const int kHeight = 1080;
+ const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
+ // Works on screenshare mode.
+ ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
+ // We rely on the automatic resolution adaptation, but we handle framerate
+ // adaptation manually by mocking the stats proxy.
+ video_source_.set_adaptation_enabled(true);
+
+ // BALANCED degradation preference is required for this feature.
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_stream_encoder_->SetSource(&video_source_,
+ webrtc::DegradationPreference::BALANCED);
+ EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
+
+ VideoFrame frame = CreateFrame(1, kWidth, kHeight);
+ frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
+
+ // Pass enough frames with the full update to trigger animation detection.
+ for (int i = 0; i < kNumFrames; ++i) {
+ int64_t timestamp_ms = CurrentTimeMs();
+ frame.set_ntp_time_ms(timestamp_ms);
+ frame.set_timestamp_us(timestamp_ms * 1000);
+ video_source_.IncomingCapturedFrame(frame);
+ WaitForEncodedFrame(timestamp_ms);
+ }
+
+ // Resolution should be limited.
+ rtc::VideoSinkWants expected;
+ expected.max_framerate_fps = kFramerateFps;
+ expected.max_pixel_count = 1280 * 720 + 1;
+ EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
+
+ // Pass one frame with no known update.
+ // Resolution cap should be removed immediately.
+ int64_t timestamp_ms = CurrentTimeMs();
+ frame.set_ntp_time_ms(timestamp_ms);
+ frame.set_timestamp_us(timestamp_ms * 1000);
+ frame.clear_update_rect();
+
+ video_source_.IncomingCapturedFrame(frame);
+ WaitForEncodedFrame(timestamp_ms);
+
+ // Resolution should be unlimited now.
+ EXPECT_THAT(video_source_.sink_wants(),
+ FpsMatchesResolutionMax(Eq(kFramerateFps)));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
+ const int kWidth = 720; // 540p adapted down.
+ const int kHeight = 405;
+ const int kNumFrames = 3;
+ // Works on screenshare mode.
+ ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
+ /*num_spatial_layers=*/2, /*screenshare=*/true);
+
+ video_source_.set_adaptation_enabled(true);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ VideoFrame frame = CreateFrame(1, kWidth, kHeight);
+
+ // Pass enough frames with the full update to trigger animation detection.
+ for (int i = 0; i < kNumFrames; ++i) {
+ int64_t timestamp_ms = CurrentTimeMs();
+ frame.set_ntp_time_ms(timestamp_ms);
+ frame.set_timestamp_us(timestamp_ms * 1000);
+ video_source_.IncomingCapturedFrame(frame);
+ WaitForEncodedFrame(timestamp_ms);
+ }
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
+ const float downscale_factors[] = {4.0, 2.0, 1.0};
+ const int number_layers =
+ sizeof(downscale_factors) / sizeof(downscale_factors[0]);
+ VideoEncoderConfig config;
+ webrtc::VideoEncoder::EncoderInfo encoder_info;
+ test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
+ for (int i = 0; i < number_layers; ++i) {
+ config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
+ config.simulcast_layers[i].active = true;
+ }
+ config.video_stream_factory =
+ rtc::make_ref_counted<cricket::EncoderStreamFactory>(
+ "VP8", /*max qp*/ 56, /*screencast*/ false,
+ /*screenshare enabled*/ false, encoder_info);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
+ 0, 0, 0);
+
+ // First initialization.
+ // Encoder should be initialized. Next frame should be key frame.
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+ sink_.SetNumExpectedLayers(number_layers);
+ int64_t timestamp_ms = kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
+ VideoFrameType::kVideoFrameKey,
+ VideoFrameType::kVideoFrameKey}));
+
+ // Disable top layer.
+ // Encoder shouldn't be re-initialized. Next frame should be delta frame.
+ config.simulcast_layers[number_layers - 1].active = false;
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+ sink_.SetNumExpectedLayers(number_layers - 1);
+ timestamp_ms += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
+ VideoFrameType::kVideoFrameDelta,
+ VideoFrameType::kVideoFrameDelta}));
+
+ // Re-enable top layer.
+ // Encoder should be re-initialized. Next frame should be key frame.
+ config.simulcast_layers[number_layers - 1].active = true;
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+ sink_.SetNumExpectedLayers(number_layers);
+ timestamp_ms += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
+ VideoFrameType::kVideoFrameKey,
+ VideoFrameType::kVideoFrameKey}));
+
+ // Top layer max rate change.
+ // Encoder shouldn't be re-initialized. Next frame should be delta frame.
+ config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+ sink_.SetNumExpectedLayers(number_layers);
+ timestamp_ms += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
+ VideoFrameType::kVideoFrameDelta,
+ VideoFrameType::kVideoFrameDelta}));
+
+ // Top layer resolution change.
+ // Encoder should be re-initialized. Next frame should be key frame.
+ config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+ sink_.SetNumExpectedLayers(number_layers);
+ timestamp_ms += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
+ WaitForEncodedFrame(timestamp_ms);
+ EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
+ EXPECT_THAT(fake_encoder_.LastFrameTypes(),
+ ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
+ VideoFrameType::kVideoFrameKey,
+ VideoFrameType::kVideoFrameKey}));
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+
+ SetUp();
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ // Capturing a frame should reconfigure the encoder and expose the encoder
+ // resolution, which is the same as the input frame.
+ int64_t timestamp_ms = kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_THAT(video_source_.sink_wants().resolutions,
+ ::testing::ElementsAreArray(
+ {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
+ // Pick downscale factors such that we never encode at full resolution - this
+ // is an interesting use case. The frame resolution influences the encoder
+ // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
+ // encoder should not ask for the frame resolution. This allows video frames
+ // to have the appearence of one resolution but optimize its internal buffers
+ // for what is actually encoded.
+ const size_t kNumSimulcastLayers = 3u;
+ const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+ const rtc::VideoSinkWants::FrameSize kLayer0Size(
+ kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
+ const rtc::VideoSinkWants::FrameSize kLayer1Size(
+ kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
+ const rtc::VideoSinkWants::FrameSize kLayer2Size(
+ kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
+
+ VideoEncoderConfig config;
+ webrtc::VideoEncoder::EncoderInfo encoder_info;
+ test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
+ for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
+ config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
+ config.simulcast_layers[i].active = true;
+ }
+ config.video_stream_factory =
+ rtc::make_ref_counted<cricket::EncoderStreamFactory>(
+ "VP8", /*max qp*/ 56, /*screencast*/ false,
+ /*screenshare enabled*/ false, encoder_info);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
+ 0, 0, 0);
+
+ // Capture a frame with all layers active.
+ int64_t timestamp_ms = kFrameIntervalMs;
+ sink_.SetNumExpectedLayers(kNumSimulcastLayers);
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(timestamp_ms);
+ // Expect encoded resolutions to match the expected simulcast layers.
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_THAT(
+ video_source_.sink_wants().resolutions,
+ ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
+
+ // Capture a frame with one of the layers inactive.
+ timestamp_ms += kFrameIntervalMs;
+ config.simulcast_layers[2].active = false;
+ sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(timestamp_ms);
+
+ // Expect encoded resolutions to match the expected simulcast layers.
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_THAT(video_source_.sink_wants().resolutions,
+ ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
+
+ // Capture a frame with all but one layer turned off.
+ timestamp_ms += kFrameIntervalMs;
+ config.simulcast_layers[1].active = false;
+ sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(timestamp_ms);
+
+ // Expect encoded resolutions to match the expected simulcast layers.
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_THAT(video_source_.sink_wants().resolutions,
+ ::testing::ElementsAreArray({kLayer0Size}));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
+ ResetEncoder("VP8", 1, 1, 1, false);
+
+ // Force encoder reconfig.
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(1, codec_width_, codec_height_));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ // Set QP on encoded frame and pass the frame to encode complete callback.
+ // Since QP is present QP parsing won't be triggered and the original value
+ // should be kept.
+ EncodedImage encoded_image;
+ encoded_image.qp_ = 123;
+ encoded_image.SetEncodedData(EncodedImageBuffer::Create(
+ kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
+ CodecSpecificInfo codec_info;
+ codec_info.codecType = kVideoCodecVP8;
+ fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
+ EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
+ EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
+ ResetEncoder("VP8", 1, 1, 1, false);
+
+ // Force encoder reconfig.
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(1, codec_width_, codec_height_));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ // Pass an encoded frame without QP to encode complete callback. QP should be
+ // parsed and set.
+ EncodedImage encoded_image;
+ encoded_image.qp_ = -1;
+ encoded_image.SetEncodedData(EncodedImageBuffer::Create(
+ kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
+ CodecSpecificInfo codec_info;
+ codec_info.codecType = kVideoCodecVP8;
+ fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
+ EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
+ EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
+ webrtc::test::ScopedKeyValueConfig field_trials(
+ field_trials_, "WebRTC-QpParsingKillSwitch/Enabled/");
+
+ ResetEncoder("VP8", 1, 1, 1, false);
+
+ // Force encoder reconfig.
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(1, codec_width_, codec_height_));
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ EncodedImage encoded_image;
+ encoded_image.qp_ = -1;
+ encoded_image.SetEncodedData(EncodedImageBuffer::Create(
+ kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
+ CodecSpecificInfo codec_info;
+ codec_info.codecType = kVideoCodecVP8;
+ fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
+ EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
+ EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ QualityScalingNotAllowed_QualityScalingDisabled) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Disable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = false;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(true);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = false;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(true);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = false;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(false);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = false;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(false);
+
+ const int MinEncBitrateKbps = 30;
+ const int MaxEncBitrateKbps = 100;
+ const int MinStartBitrateKbp = 50;
+ const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
+ /*frame_size_pixels=*/codec_width_ * codec_height_,
+ /*min_start_bitrate_bps=*/MinStartBitrateKbp,
+ /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
+ /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
+ video_encoder_config.simulcast_layers[0].min_bitrate_bps =
+ MinEncBitrateKbps * 1000;
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(
+ MaxEncBitrateKbps,
+ static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
+ EXPECT_EQ(
+ MinEncBitrateKbps,
+ static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(false);
+
+ absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
+ EncoderInfoSettings::
+ GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
+ codec_width_ * codec_height_,
+ EncoderInfoSettings::
+ GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
+ EXPECT_TRUE(suitable_bitrate_limit.has_value());
+
+ const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
+ const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
+ const int TargetEncBitrate = MaxEncBitrate;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(TargetEncBitrate),
+ DataRate::BitsPerSec(TargetEncBitrate),
+ DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = MaxEncBitrate;
+ video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(
+ MaxEncBitrate / 1000,
+ static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
+ EXPECT_EQ(
+ MinEncBitrate / 1000,
+ static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, NormalComplexityWithMoreThanTwoCores) {
+ ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
+ /*num_spatial_layers=*/1,
+ /*screenshare=*/false, /*allocation_callback_type=*/
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoBitrateAllocationWhenScreenSharing,
+ /*num_cores=*/3);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(1, /*width=*/320, /*height=*/180));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
+ VideoCodecComplexity::kComplexityNormal);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ NormalComplexityWhenLowTierOptimizationsAreDisabled) {
+ webrtc::test::ScopedKeyValueConfig field_trials(
+ field_trials_, "WebRTC-VP9-LowTierOptimizations/Disabled/");
+
+ ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
+ /*num_spatial_layers=*/1,
+ /*screenshare=*/false, /*allocation_callback_type=*/
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoBitrateAllocationWhenScreenSharing,
+ /*num_cores=*/2);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(1, /*width=*/320, /*height=*/180));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
+ VideoCodecComplexity::kComplexityNormal);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, LowComplexityWithTwoCores) {
+ ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
+ /*num_spatial_layers=*/1,
+ /*screenshare=*/false, /*allocation_callback_type=*/
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoBitrateAllocationWhenScreenSharing,
+ /*num_cores=*/2);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(1, /*width=*/320, /*height=*/180));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
+ VideoCodecComplexity::kComplexityLow);
+ video_stream_encoder_->Stop();
+}
+
+#if !defined(WEBRTC_IOS)
+// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
+// disabled by default on iOS.
+TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = true;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(true);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = true;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP not trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(false);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = true;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ // When quality_scaler doesn't work and is_quality_scaling_allowed is
+ // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(true);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = true;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ // bandwidth_quality_scaler isn't working, but quality_scaler is working.
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(false);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = true;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ RequestsRefreshFrameAfterEarlyDroppedNativeFrame) {
+ // Send a native frame before encoder rates have been set. The encoder is
+ // seen as paused at this time.
+ rtc::Event frame_destroyed_event;
+ video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
+ /*ntp_time_ms=*/1, &frame_destroyed_event, codec_width_, codec_height_));
+
+ // Frame should be dropped and destroyed.
+ ExpectDroppedFrame();
+ EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
+ EXPECT_EQ(video_source_.refresh_frames_requested_, 0);
+
+ // Set bitrates, unpausing the encoder and triggering a request for a refresh
+ // frame.
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+ EXPECT_EQ(video_source_.refresh_frames_requested_, 1);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, RecreatesEncoderWhenEnableVp9SpatialLayer) {
+ // Set up encoder to use VP9 SVC using two spatial layers.
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
+ fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
+ /* num_streams*/ 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+ VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
+ vp9_settings.numberOfSpatialLayers = 2;
+ vp9_settings.numberOfTemporalLayers = 2;
+ vp9_settings.interLayerPred = InterLayerPredMode::kOn;
+ vp9_settings.automaticResizeOn = false;
+ video_encoder_config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
+ vp9_settings);
+ video_encoder_config.spatial_layers = GetSvcConfig(1280, 720,
+ /*fps=*/30.0,
+ /*first_active_layer=*/0,
+ /*num_spatial_layers=*/2,
+ /*num_temporal_layers=*/3,
+ /*is_screenshare=*/false);
+ ConfigureEncoder(video_encoder_config.Copy(),
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoLayersAllocation);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(fake_encoder_.GetNumInitializations(), 1);
+
+ // Turn off the top spatial layer. This does not require an encoder reset.
+ video_encoder_config.spatial_layers[1].active = false;
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength, nullptr);
+
+ time_controller_.AdvanceTime(TimeDelta::Millis(33));
+ video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(fake_encoder_.GetNumInitializations(), 1);
+
+ // Turn on the top spatial layer again, this does require an encoder reset.
+ video_encoder_config.spatial_layers[1].active = true;
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength, nullptr);
+
+ time_controller_.AdvanceTime(TimeDelta::Millis(33));
+ video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
+ WaitForEncodedFrame(CurrentTimeMs());
+ EXPECT_EQ(fake_encoder_.GetNumInitializations(), 2);
+
+ video_stream_encoder_->Stop();
+}
+
+#endif // !defined(WEBRTC_IOS)
+
+// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
+class VideoStreamEncoderWithRealEncoderTest
+ : public VideoStreamEncoderTest,
+ public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
+ public:
+ VideoStreamEncoderWithRealEncoderTest()
+ : VideoStreamEncoderTest(),
+ codec_type_(std::get<0>(GetParam())),
+ allow_i420_conversion_(std::get<1>(GetParam())) {}
+
+ void SetUp() override {
+ VideoStreamEncoderTest::SetUp();
+ std::unique_ptr<VideoEncoder> encoder;
+ switch (codec_type_) {
+ case kVideoCodecVP8:
+ encoder = VP8Encoder::Create();
+ break;
+ case kVideoCodecVP9:
+ encoder = VP9Encoder::Create();
+ break;
+ case kVideoCodecAV1:
+ encoder = CreateLibaomAv1Encoder();
+ break;
+ case kVideoCodecH264:
+ encoder =
+ H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
+ break;
+ case kVideoCodecMultiplex:
+ mock_encoder_factory_for_multiplex_ =
+ std::make_unique<MockVideoEncoderFactory>();
+ EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
+ EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
+ .WillRepeatedly([] { return VP8Encoder::Create(); });
+ encoder = std::make_unique<MultiplexEncoderAdapter>(
+ mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
+ false);
+ break;
+ default:
+ RTC_DCHECK_NOTREACHED();
+ }
+ ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
+ }
+
+ void TearDown() override {
+ video_stream_encoder_->Stop();
+ // Ensure `video_stream_encoder_` is destroyed before
+ // `encoder_proxy_factory_`.
+ video_stream_encoder_.reset();
+ VideoStreamEncoderTest::TearDown();
+ }
+
+ protected:
+ void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
+ std::unique_ptr<VideoEncoder> encoder) {
+ // Configure VSE to use the encoder.
+ encoder_ = std::move(encoder);
+ encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
+ encoder_.get(), &encoder_selector_);
+ video_send_config_.encoder_settings.encoder_factory =
+ encoder_proxy_factory_.get();
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
+ video_encoder_config_ = video_encoder_config.Copy();
+ ConfigureEncoder(video_encoder_config_.Copy());
+
+ // Set bitrate to ensure frame is not dropped.
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
+ }
+
+ const VideoCodecType codec_type_;
+ const bool allow_i420_conversion_;
+ NiceMock<MockEncoderSelector> encoder_selector_;
+ std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
+ std::unique_ptr<VideoEncoder> encoder_;
+ std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
+};
+
+TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
+ auto native_i420_frame = test::CreateMappableNativeFrame(
+ 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
+ video_source_.IncomingCapturedFrame(native_i420_frame);
+ WaitForEncodedFrame(codec_width_, codec_height_);
+
+ auto mappable_native_buffer =
+ test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
+ std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
+ mappable_native_buffer->GetMappedFramedBuffers();
+ ASSERT_EQ(mapped_frame_buffers.size(), 1u);
+ EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
+ EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
+ EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
+}
+
+TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
+ auto native_nv12_frame = test::CreateMappableNativeFrame(
+ 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
+ video_source_.IncomingCapturedFrame(native_nv12_frame);
+ WaitForEncodedFrame(codec_width_, codec_height_);
+
+ auto mappable_native_buffer =
+ test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
+ std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
+ mappable_native_buffer->GetMappedFramedBuffers();
+ ASSERT_EQ(mapped_frame_buffers.size(), 1u);
+ EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
+ EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
+ EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
+
+ if (!allow_i420_conversion_) {
+ EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
+ }
+}
+
+TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
+ if (codec_type_ == kVideoCodecMultiplex) {
+ // Multiplex codec here uses wrapped mock codecs, ignore for this test.
+ return;
+ }
+
+ const size_t kNumSpatialLayers = 3u;
+ const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
+ const int kFrameWidth = 1280;
+ const int kFrameHeight = 720;
+ const rtc::VideoSinkWants::FrameSize kLayer0Size(
+ kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
+ const rtc::VideoSinkWants::FrameSize kLayer1Size(
+ kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
+ const rtc::VideoSinkWants::FrameSize kLayer2Size(
+ kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
+
+ VideoEncoderConfig config;
+ webrtc::VideoEncoder::EncoderInfo encoder_info;
+ if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
+ test::FillEncoderConfiguration(codec_type_, 1, &config);
+ config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
+ VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
+ vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
+ vp9_settings.numberOfTemporalLayers = 3;
+ vp9_settings.automaticResizeOn = false;
+ config.encoder_specific_settings =
+ rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
+ vp9_settings);
+ config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
+ /*fps=*/30.0,
+ /*first_active_layer=*/0,
+ /*num_spatial_layers=*/3,
+ /*num_temporal_layers=*/3,
+ /*is_screenshare=*/false);
+ } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
+ test::FillEncoderConfiguration(codec_type_, 1, &config);
+ config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
+ config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
+ /*fps=*/30.0,
+ /*first_active_layer=*/0,
+ /*num_spatial_layers=*/3,
+ /*num_temporal_layers=*/3,
+ /*is_screenshare=*/false);
+ config.simulcast_layers[0].scalability_mode = ScalabilityMode::kL3T3_KEY;
+ } else {
+ // Simulcast for VP8/H264.
+ test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
+ for (size_t i = 0; i < kNumSpatialLayers; ++i) {
+ config.simulcast_layers[i].scale_resolution_down_by =
+ kDownscaleFactors[i];
+ config.simulcast_layers[i].active = true;
+ }
+ if (codec_type_ == VideoCodecType::kVideoCodecH264) {
+ // Turn off frame dropping to prevent flakiness.
+ config.frame_drop_enabled = false;
+ }
+ }
+
+ auto set_layer_active = [&](int layer_idx, bool active) {
+ if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
+ codec_type_ == VideoCodecType::kVideoCodecAV1) {
+ config.spatial_layers[layer_idx].active = active;
+ } else {
+ config.simulcast_layers[layer_idx].active = active;
+ }
+ };
+
+ config.video_stream_factory =
+ rtc::make_ref_counted<cricket::EncoderStreamFactory>(
+ CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
+ /*screencast*/ false,
+ /*screenshare enabled*/ false, encoder_info);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
+ 0, 0, 0);
+
+ // Capture a frame with all layers active.
+ sink_.SetNumExpectedLayers(kNumSpatialLayers);
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+ int64_t timestamp_ms = kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+
+ WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ // Capture a frame with one of the layers inactive.
+ set_layer_active(2, false);
+ sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+ timestamp_ms += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
+
+ // New target bitrates signaled based on lower resolution.
+ DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ // Re-enable the top layer.
+ set_layer_active(2, true);
+ sink_.SetNumExpectedLayers(kNumSpatialLayers);
+ video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ // Bitrate target adjusted back up to enable HD layer...
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
+ DataRate::KilobitsPerSec(1800), 0, 0, 0);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ // ...then add a new frame.
+ timestamp_ms += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+ WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ video_stream_encoder_->Stop();
+}
+
+std::string TestParametersVideoCodecAndAllowI420ConversionToString(
+ testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
+ VideoCodecType codec_type = std::get<0>(info.param);
+ bool allow_i420_conversion = std::get<1>(info.param);
+ std::string str;
+ switch (codec_type) {
+ case kVideoCodecGeneric:
+ str = "Generic";
+ break;
+ case kVideoCodecVP8:
+ str = "VP8";
+ break;
+ case kVideoCodecVP9:
+ str = "VP9";
+ break;
+ case kVideoCodecAV1:
+ str = "AV1";
+ break;
+ case kVideoCodecH264:
+ str = "H264";
+ break;
+ case kVideoCodecMultiplex:
+ str = "Multiplex";
+ break;
+ default:
+ RTC_DCHECK_NOTREACHED();
+ }
+ str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
+ return str;
+}
+
+constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
+ std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
+constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
+ std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
+constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
+ std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false);
+constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
+ std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
+#if defined(WEBRTC_USE_H264)
+constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
+ std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
+
+// The windows compiler does not tolerate #if statements inside the
+// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
+// and without H264).
+INSTANTIATE_TEST_SUITE_P(
+ All,
+ VideoStreamEncoderWithRealEncoderTest,
+ ::testing::Values(kVP8DisallowConversion,
+ kVP9DisallowConversion,
+ kAV1AllowConversion,
+ kMultiplexDisallowConversion,
+ kH264AllowConversion),
+ TestParametersVideoCodecAndAllowI420ConversionToString);
+#else
+INSTANTIATE_TEST_SUITE_P(
+ All,
+ VideoStreamEncoderWithRealEncoderTest,
+ ::testing::Values(kVP8DisallowConversion,
+ kVP9DisallowConversion,
+ kAV1AllowConversion,
+ kMultiplexDisallowConversion),
+ TestParametersVideoCodecAndAllowI420ConversionToString);
+#endif
+
+class ReconfigureEncoderTest : public VideoStreamEncoderTest {
+ protected:
+ void RunTest(const std::vector<VideoStream>& configs,
+ const int expected_num_init_encode) {
+ ConfigureEncoder(configs[0]);
+ OnBitrateUpdated(kTargetBitrate);
+ InsertFrameAndWaitForEncoded();
+ EXPECT_EQ(1, sink_.number_of_reconfigurations());
+ ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
+ EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
+ ExpectEqual(fake_encoder_.config(), configs[0]);
+
+ // Reconfigure encoder, the encoder should only be reconfigured if needed.
+ ConfigureEncoder(configs[1]);
+ InsertFrameAndWaitForEncoded();
+ EXPECT_EQ(2, sink_.number_of_reconfigurations());
+ ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
+ EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
+ if (expected_num_init_encode > 1)
+ ExpectEqual(fake_encoder_.config(), configs[1]);
+
+ video_stream_encoder_->Stop();
+ }
+
+ void ConfigureEncoder(const VideoStream& stream) {
+ VideoEncoderConfig config;
+ webrtc::VideoEncoder::EncoderInfo encoder_info;
+
+ test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
+ config.max_bitrate_bps = stream.max_bitrate_bps;
+ config.simulcast_layers[0] = stream;
+ config.video_stream_factory =
+ rtc::make_ref_counted<cricket::EncoderStreamFactory>(
+ /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
+ /*conference_mode=*/false, encoder_info);
+ video_stream_encoder_->ConfigureEncoder(std::move(config),
+ kMaxPayloadLength);
+ }
+
+ void OnBitrateUpdated(DataRate bitrate) {
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ bitrate, bitrate, bitrate, 0, 0, 0);
+ }
+
+ void InsertFrameAndWaitForEncoded() {
+ timestamp_ms_ += kFrameIntervalMs;
+ video_source_.IncomingCapturedFrame(
+ CreateFrame(timestamp_ms_, kWidth, kHeight));
+ sink_.WaitForEncodedFrame(timestamp_ms_);
+ }
+
+ void ExpectEqual(const VideoCodec& actual,
+ const VideoStream& expected) const {
+ EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
+ EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
+ EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
+ static_cast<unsigned int>(expected.min_bitrate_bps));
+ EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
+ static_cast<unsigned int>(expected.max_bitrate_bps));
+ EXPECT_EQ(actual.simulcastStream[0].width,
+ kWidth / expected.scale_resolution_down_by);
+ EXPECT_EQ(actual.simulcastStream[0].height,
+ kHeight / expected.scale_resolution_down_by);
+ EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
+ expected.num_temporal_layers);
+ EXPECT_EQ(actual.GetScalabilityMode(), expected.scalability_mode);
+ }
+
+ VideoStream DefaultConfig() const {
+ VideoStream stream;
+ stream.max_framerate = 25;
+ stream.min_bitrate_bps = 35000;
+ stream.max_bitrate_bps = 900000;
+ stream.scale_resolution_down_by = 1.0;
+ stream.num_temporal_layers = 1;
+ stream.bitrate_priority = 1.0;
+ stream.scalability_mode = absl::nullopt;
+ return stream;
+ }
+
+ const int kWidth = 640;
+ const int kHeight = 360;
+ int64_t timestamp_ms_ = 0;
+};
+
+TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
+ VideoStream config1 = DefaultConfig();
+ VideoStream config2 = config1;
+ config2.max_framerate++;
+
+ RunTest({config1, config2}, /*expected_num_init_encode=*/1);
+}
+
+TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
+ VideoStream config1 = DefaultConfig();
+ VideoStream config2 = config1;
+ config2.min_bitrate_bps += 10000;
+
+ RunTest({config1, config2}, /*expected_num_init_encode=*/1);
+}
+
+TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
+ VideoStream config1 = DefaultConfig();
+ VideoStream config2 = config1;
+ config2.max_bitrate_bps += 100000;
+
+ RunTest({config1, config2}, /*expected_num_init_encode=*/1);
+}
+
+TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
+ VideoStream config1 = DefaultConfig();
+ VideoStream config2 = config1;
+ config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
+
+ RunTest({config1, config2}, /*expected_num_init_encode=*/1);
+}
+
+TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
+ VideoStream config1 = DefaultConfig();
+ VideoStream config2 = config1;
+ config2.scale_resolution_down_by *= 2;
+
+ RunTest({config1, config2}, /*expected_num_init_encode=*/2);
+}
+
+TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
+ VideoStream config1 = DefaultConfig();
+ VideoStream config2 = config1;
+ config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
+
+ RunTest({config1, config2}, /*expected_num_init_encode=*/2);
+}
+
+TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
+ VideoStream config1 = DefaultConfig();
+ VideoStream config2 = config1;
+ config2.scalability_mode = ScalabilityMode::kL2T1;
+
+ RunTest({config1, config2}, /*expected_num_init_encode=*/2);
+}
+
+TEST_F(ReconfigureEncoderTest,
+ UpdatesNumTemporalLayersFromScalabilityModeChanges) {
+ VideoStream config1 = DefaultConfig();
+ VideoStream config2 = config1;
+ config2.scalability_mode = ScalabilityMode::kL1T2;
+ config2.num_temporal_layers = 2;
+
+ RunTest({config1, config2}, /*expected_num_init_encode=*/2);
+}
+
+// Simple test that just creates and then immediately destroys an encoder.
+// The purpose of the test is to make sure that nothing bad happens if the
+// initialization step on the encoder queue, doesn't run.
+TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
+ class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
+ public:
+ SuperLazyTaskQueue() = default;
+ ~SuperLazyTaskQueue() override = default;
+
+ private:
+ void Delete() override { delete this; }
+ void PostTask(absl::AnyInvocable<void() &&> task) override {
+ // meh.
+ }
+ void PostDelayedTask(absl::AnyInvocable<void() &&> task,
+ TimeDelta delay) override {
+ ASSERT_TRUE(false);
+ }
+ void PostDelayedHighPrecisionTask(absl::AnyInvocable<void() &&> task,
+ TimeDelta delay) override {
+ ADD_FAILURE();
+ }
+ };
+
+ // Lots of boiler plate.
+ test::ScopedKeyValueConfig field_trials;
+ GlobalSimulatedTimeController time_controller(Timestamp::Zero());
+ auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
+ time_controller.GetClock(), VideoSendStream::Config(nullptr),
+ webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, field_trials);
+ SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
+ time_controller.GetClock());
+ test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
+ std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
+ CreateBuiltinVideoBitrateAllocatorFactory();
+ VideoStreamEncoderSettings encoder_settings{
+ VideoEncoder::Capabilities(/*loss_notification=*/false)};
+ encoder_settings.encoder_factory = &encoder_factory;
+ encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
+
+ auto adapter = std::make_unique<MockFrameCadenceAdapter>();
+ EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
+
+ std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
+ encoder_queue(new SuperLazyTaskQueue());
+
+ // Construct a VideoStreamEncoder instance and let it go out of scope without
+ // doing anything else (including calling Stop()). This should be fine since
+ // the posted init task will simply be deleted.
+ auto encoder = std::make_unique<VideoStreamEncoder>(
+ time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
+ std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get(),
+ field_trials),
+ std::move(adapter), std::move(encoder_queue),
+ VideoStreamEncoder::BitrateAllocationCallbackType::
+ kVideoBitrateAllocation,
+ field_trials);
+
+ // Stop the encoder explicitly. This additional step tests if we could
+ // hang when calling stop and the TQ has been stopped and/or isn't accepting
+ // any more tasks.
+ encoder->Stop();
+}
+
+TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
+ auto adapter = std::make_unique<MockFrameCadenceAdapter>();
+ auto* adapter_ptr = adapter.get();
+ SimpleVideoStreamEncoderFactory factory;
+ FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
+ nullptr;
+ EXPECT_CALL(*adapter_ptr, Initialize)
+ .WillOnce(Invoke([&video_stream_encoder_callback](
+ FrameCadenceAdapterInterface::Callback* callback) {
+ video_stream_encoder_callback = callback;
+ }));
+ TaskQueueBase* encoder_queue = nullptr;
+ auto video_stream_encoder =
+ factory.Create(std::move(adapter), &encoder_queue);
+
+ // First a call before we know the frame size and hence cannot compute the
+ // number of simulcast layers.
+ EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
+ &FrameCadenceAdapterInterface::
+ ZeroHertzModeParams::num_simulcast_layers,
+ Eq(0u)))));
+ VideoEncoderConfig config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
+ config.content_type = VideoEncoderConfig::ContentType::kScreen;
+ video_stream_encoder->ConfigureEncoder(std::move(config), 0);
+ factory.DepleteTaskQueues();
+
+ // Then a call as we've computed the number of simulcast layers after a passed
+ // frame.
+ EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
+ &FrameCadenceAdapterInterface::
+ ZeroHertzModeParams::num_simulcast_layers,
+ Gt(0u)))));
+ PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
+ factory.DepleteTaskQueues();
+ Mock::VerifyAndClearExpectations(adapter_ptr);
+
+ // Expect a disabled zero-hertz mode after passing realtime video.
+ EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
+ VideoEncoderConfig config2;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
+ config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
+ video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
+ PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
+ factory.DepleteTaskQueues();
+}
+
+TEST(VideoStreamEncoderFrameCadenceTest,
+ ForwardsFramesIntoFrameCadenceAdapter) {
+ auto adapter = std::make_unique<MockFrameCadenceAdapter>();
+ auto* adapter_ptr = adapter.get();
+ test::FrameForwarder video_source;
+ SimpleVideoStreamEncoderFactory factory;
+ auto video_stream_encoder = factory.Create(std::move(adapter));
+ video_stream_encoder->SetSource(
+ &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+
+ EXPECT_CALL(*adapter_ptr, OnFrame);
+ auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
+ video_source.IncomingCapturedFrame(
+ VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
+}
+
+TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
+ auto adapter = std::make_unique<MockFrameCadenceAdapter>();
+ auto* adapter_ptr = adapter.get();
+ test::FrameForwarder video_source;
+ SimpleVideoStreamEncoderFactory factory;
+ FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
+ nullptr;
+ EXPECT_CALL(*adapter_ptr, Initialize)
+ .WillOnce(Invoke([&video_stream_encoder_callback](
+ FrameCadenceAdapterInterface::Callback* callback) {
+ video_stream_encoder_callback = callback;
+ }));
+ TaskQueueBase* encoder_queue = nullptr;
+ auto video_stream_encoder =
+ factory.Create(std::move(adapter), &encoder_queue);
+
+ // This is just to make the VSE operational. We'll feed a frame directly by
+ // the callback interface.
+ video_stream_encoder->SetSource(
+ &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
+ video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
+ /*max_data_payload_length=*/1000);
+
+ EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
+ EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
+ PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
+ factory.DepleteTaskQueues();
+}
+
+TEST(VideoStreamEncoderFrameCadenceTest,
+ DeactivatesActivatesLayersOnBitrateChanges) {
+ auto adapter = std::make_unique<MockFrameCadenceAdapter>();
+ auto* adapter_ptr = adapter.get();
+ SimpleVideoStreamEncoderFactory factory;
+ FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
+ nullptr;
+ EXPECT_CALL(*adapter_ptr, Initialize)
+ .WillOnce(Invoke([&video_stream_encoder_callback](
+ FrameCadenceAdapterInterface::Callback* callback) {
+ video_stream_encoder_callback = callback;
+ }));
+ TaskQueueBase* encoder_queue = nullptr;
+ auto video_stream_encoder =
+ factory.Create(std::move(adapter), &encoder_queue);
+
+ // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
+ // {150000, 450000}.
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
+ video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+ // Ensure an encoder is created.
+ PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
+
+ // Both layers enabled at 1 MBit/s.
+ video_stream_encoder->OnBitrateUpdated(
+ DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
+ DataRate::KilobitsPerSec(1000), 0, 0, 0);
+ EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
+ EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
+ factory.DepleteTaskQueues();
+ Mock::VerifyAndClearExpectations(adapter_ptr);
+
+ // Layer 1 disabled at 200 KBit/s.
+ video_stream_encoder->OnBitrateUpdated(
+ DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
+ DataRate::KilobitsPerSec(200), 0, 0, 0);
+ EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
+ EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
+ factory.DepleteTaskQueues();
+ Mock::VerifyAndClearExpectations(adapter_ptr);
+
+ // All layers off at suspended video.
+ video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
+ DataRate::Zero(), 0, 0, 0);
+ EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
+ EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
+ factory.DepleteTaskQueues();
+ Mock::VerifyAndClearExpectations(adapter_ptr);
+
+ // Both layers enabled again back at 1 MBit/s.
+ video_stream_encoder->OnBitrateUpdated(
+ DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
+ DataRate::KilobitsPerSec(1000), 0, 0, 0);
+ EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
+ EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
+ factory.DepleteTaskQueues();
+}
+
+TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
+ auto adapter = std::make_unique<MockFrameCadenceAdapter>();
+ auto* adapter_ptr = adapter.get();
+ SimpleVideoStreamEncoderFactory factory;
+ FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
+ nullptr;
+ EXPECT_CALL(*adapter_ptr, Initialize)
+ .WillOnce(Invoke([&video_stream_encoder_callback](
+ FrameCadenceAdapterInterface::Callback* callback) {
+ video_stream_encoder_callback = callback;
+ }));
+ TaskQueueBase* encoder_queue = nullptr;
+ auto video_stream_encoder =
+ factory.Create(std::move(adapter), &encoder_queue);
+
+ // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
+ video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+ video_stream_encoder->OnBitrateUpdated(
+ DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
+ DataRate::KilobitsPerSec(1000), 0, 0, 0);
+
+ // Pass a frame which has unconverged results.
+ PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
+ EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
+ .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
+ rtc::scoped_refptr<EncodedImageBuffer> buffer) {
+ encoded_image.qp_ = kVp8SteadyStateQpThreshold + 1;
+ CodecSpecificInfo codec_specific;
+ codec_specific.codecType = kVideoCodecVP8;
+ return codec_specific;
+ }));
+ EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
+ EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
+ factory.DepleteTaskQueues();
+ Mock::VerifyAndClearExpectations(adapter_ptr);
+ Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
+
+ // Pass a frame which converges in layer 0 and not in layer 1.
+ PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
+ EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
+ .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
+ rtc::scoped_refptr<EncodedImageBuffer> buffer) {
+ // This sets spatial index 0 content to be at target quality, while
+ // index 1 content is not.
+ encoded_image.qp_ = kVp8SteadyStateQpThreshold +
+ (encoded_image.SpatialIndex() == 0 ? 0 : 1);
+ CodecSpecificInfo codec_specific;
+ codec_specific.codecType = kVideoCodecVP8;
+ return codec_specific;
+ }));
+ EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
+ EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
+ factory.DepleteTaskQueues();
+ Mock::VerifyAndClearExpectations(adapter_ptr);
+ Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
+}
+
+TEST(VideoStreamEncoderFrameCadenceTest,
+ RequestsRefreshFramesWhenCadenceAdapterInstructs) {
+ auto adapter = std::make_unique<MockFrameCadenceAdapter>();
+ auto* adapter_ptr = adapter.get();
+ MockVideoSourceInterface mock_source;
+ SimpleVideoStreamEncoderFactory factory;
+ FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
+ nullptr;
+ EXPECT_CALL(*adapter_ptr, Initialize)
+ .WillOnce(Invoke([&video_stream_encoder_callback](
+ FrameCadenceAdapterInterface::Callback* callback) {
+ video_stream_encoder_callback = callback;
+ }));
+ TaskQueueBase* encoder_queue = nullptr;
+ auto video_stream_encoder =
+ factory.Create(std::move(adapter), &encoder_queue);
+ video_stream_encoder->SetSource(
+ &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ VideoEncoderConfig config;
+ config.content_type = VideoEncoderConfig::ContentType::kScreen;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
+ video_stream_encoder->ConfigureEncoder(std::move(config), 0);
+ PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
+ // Ensure the encoder is set up.
+ factory.DepleteTaskQueues();
+
+ EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
+ .WillOnce(Invoke([video_stream_encoder_callback] {
+ video_stream_encoder_callback->RequestRefreshFrame();
+ }));
+ EXPECT_CALL(mock_source, RequestRefreshFrame);
+ video_stream_encoder->SendKeyFrame();
+ factory.DepleteTaskQueues();
+ Mock::VerifyAndClearExpectations(adapter_ptr);
+ Mock::VerifyAndClearExpectations(&mock_source);
+
+ EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
+ EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
+ video_stream_encoder->SendKeyFrame();
+ factory.DepleteTaskQueues();
+}
+
+TEST(VideoStreamEncoderFrameCadenceTest,
+ RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
+ SimpleVideoStreamEncoderFactory factory;
+ auto encoder_queue =
+ factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
+ "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
+
+ // Enables zero-hertz mode.
+ test::ScopedKeyValueConfig field_trials(
+ "WebRTC-ZeroHertzScreenshare/Enabled/");
+ auto adapter = FrameCadenceAdapterInterface::Create(
+ factory.GetTimeController()->GetClock(), encoder_queue.get(),
+ field_trials);
+ FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
+
+ MockVideoSourceInterface mock_source;
+ auto video_stream_encoder = factory.CreateWithEncoderQueue(
+ std::move(adapter), std::move(encoder_queue), &field_trials);
+
+ video_stream_encoder->SetSource(
+ &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ VideoEncoderConfig config;
+ config.content_type = VideoEncoderConfig::ContentType::kScreen;
+ test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
+ video_stream_encoder->ConfigureEncoder(std::move(config), 0);
+
+ // Eventually expect a refresh frame request when requesting a key frame
+ // before initializing zero-hertz mode. This can happen in reality because the
+ // threads invoking key frame requests and constraints setup aren't
+ // synchronized.
+ EXPECT_CALL(mock_source, RequestRefreshFrame);
+ video_stream_encoder->SendKeyFrame();
+ constexpr int kMaxFps = 30;
+ adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFps});
+ factory.GetTimeController()->AdvanceTime(
+ TimeDelta::Seconds(1) *
+ FrameCadenceAdapterInterface::kOnDiscardedFrameRefreshFramePeriod /
+ kMaxFps);
+}
+
+} // namespace webrtc