diff options
Diffstat (limited to 'third_party/libwebrtc/modules/video_coding/utility')
39 files changed, 7317 insertions, 0 deletions
diff --git a/third_party/libwebrtc/modules/video_coding/utility/bandwidth_quality_scaler.cc b/third_party/libwebrtc/modules/video_coding/utility/bandwidth_quality_scaler.cc new file mode 100644 index 0000000000..13502a142b --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/bandwidth_quality_scaler.cc @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/bandwidth_quality_scaler.h" + +#include <algorithm> +#include <memory> +#include <utility> +#include <vector> + +#include "api/video/video_adaptation_reason.h" +#include "api/video_codecs/video_encoder.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/bandwidth_quality_scaler_settings.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/exp_filter.h" +#include "rtc_base/time_utils.h" +#include "rtc_base/weak_ptr.h" + +namespace webrtc { + +namespace { + +constexpr int kDefaultMaxWindowSizeMs = 5000; +constexpr float kHigherMaxBitrateTolerationFactor = 0.95; +constexpr float kLowerMinBitrateTolerationFactor = 0.8; +constexpr int kDefaultBitrateStateUpdateIntervalSeconds = 5; +} // namespace + +BandwidthQualityScaler::BandwidthQualityScaler( + BandwidthQualityScalerUsageHandlerInterface* handler) + : kBitrateStateUpdateInterval(TimeDelta::Seconds( + BandwidthQualityScalerSettings::ParseFromFieldTrials() + .BitrateStateUpdateInterval() + .value_or(kDefaultBitrateStateUpdateIntervalSeconds))), + handler_(handler), + encoded_bitrate_(kDefaultMaxWindowSizeMs, RateStatistics::kBpsScale), + weak_ptr_factory_(this) { + RTC_DCHECK_RUN_ON(&task_checker_); + RTC_DCHECK(handler_ != nullptr); + + StartCheckForBitrate(); +} + +BandwidthQualityScaler::~BandwidthQualityScaler() { + RTC_DCHECK_RUN_ON(&task_checker_); +} + +void BandwidthQualityScaler::StartCheckForBitrate() { + RTC_DCHECK_RUN_ON(&task_checker_); + TaskQueueBase::Current()->PostDelayedTask( + [this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] { + if (!this_weak_ptr) { + // The caller BandwidthQualityScaler has been deleted. + return; + } + RTC_DCHECK_RUN_ON(&task_checker_); + switch (CheckBitrate()) { + case BandwidthQualityScaler::CheckBitrateResult::kHighBitRate: { + handler_->OnReportUsageBandwidthHigh(); + last_frame_size_pixels_.reset(); + break; + } + case BandwidthQualityScaler::CheckBitrateResult::kLowBitRate: { + handler_->OnReportUsageBandwidthLow(); + last_frame_size_pixels_.reset(); + break; + } + case BandwidthQualityScaler::CheckBitrateResult::kNormalBitrate: { + break; + } + case BandwidthQualityScaler::CheckBitrateResult:: + kInsufficientSamples: { + break; + } + } + StartCheckForBitrate(); + }, + kBitrateStateUpdateInterval); +} + +void BandwidthQualityScaler::ReportEncodeInfo(int frame_size_bytes, + int64_t time_sent_in_ms, + uint32_t encoded_width, + uint32_t encoded_height) { + RTC_DCHECK_RUN_ON(&task_checker_); + last_time_sent_in_ms_ = time_sent_in_ms; + last_frame_size_pixels_ = encoded_width * encoded_height; + encoded_bitrate_.Update(frame_size_bytes, time_sent_in_ms); +} + +void BandwidthQualityScaler::SetResolutionBitrateLimits( + const std::vector<VideoEncoder::ResolutionBitrateLimits>& + resolution_bitrate_limits) { + if (resolution_bitrate_limits.empty()) { + resolution_bitrate_limits_ = EncoderInfoSettings:: + GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted(); + } else { + resolution_bitrate_limits_ = resolution_bitrate_limits; + } +} + +BandwidthQualityScaler::CheckBitrateResult +BandwidthQualityScaler::CheckBitrate() { + RTC_DCHECK_RUN_ON(&task_checker_); + if (!last_frame_size_pixels_.has_value() || + !last_time_sent_in_ms_.has_value()) { + return BandwidthQualityScaler::CheckBitrateResult::kInsufficientSamples; + } + + absl::optional<int64_t> current_bitrate_bps = + encoded_bitrate_.Rate(last_time_sent_in_ms_.value()); + if (!current_bitrate_bps.has_value()) { + // We can't get a valid bitrate due to not enough data points. + return BandwidthQualityScaler::CheckBitrateResult::kInsufficientSamples; + } + absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit = + EncoderInfoSettings:: + GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted( + last_frame_size_pixels_, resolution_bitrate_limits_); + + if (!suitable_bitrate_limit.has_value()) { + return BandwidthQualityScaler::CheckBitrateResult::kInsufficientSamples; + } + + // Multiply by toleration factor to solve the frequent adaptation due to + // critical value. + if (current_bitrate_bps > suitable_bitrate_limit->max_bitrate_bps * + kHigherMaxBitrateTolerationFactor) { + return BandwidthQualityScaler::CheckBitrateResult::kLowBitRate; + } else if (current_bitrate_bps < + suitable_bitrate_limit->min_start_bitrate_bps * + kLowerMinBitrateTolerationFactor) { + return BandwidthQualityScaler::CheckBitrateResult::kHighBitRate; + } + return BandwidthQualityScaler::CheckBitrateResult::kNormalBitrate; +} + +BandwidthQualityScalerUsageHandlerInterface:: + ~BandwidthQualityScalerUsageHandlerInterface() {} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/bandwidth_quality_scaler.h b/third_party/libwebrtc/modules/video_coding/utility/bandwidth_quality_scaler.h new file mode 100644 index 0000000000..7cd1de0dd2 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/bandwidth_quality_scaler.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_BANDWIDTH_QUALITY_SCALER_H_ +#define MODULES_VIDEO_CODING_UTILITY_BANDWIDTH_QUALITY_SCALER_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <memory> +#include <vector> + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/sequence_checker.h" +#include "api/video_codecs/video_encoder.h" +#include "rtc_base/experiments/encoder_info_settings.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/exp_filter.h" +#include "rtc_base/rate_statistics.h" +#include "rtc_base/ref_count.h" +#include "rtc_base/system/no_unique_address.h" +#include "rtc_base/weak_ptr.h" + +namespace webrtc { + +class BandwidthQualityScalerUsageHandlerInterface { + public: + virtual ~BandwidthQualityScalerUsageHandlerInterface(); + + virtual void OnReportUsageBandwidthHigh() = 0; + virtual void OnReportUsageBandwidthLow() = 0; +}; + +// BandwidthQualityScaler runs asynchronously and monitors bandwidth values of +// encoded frames. It holds a reference to a +// BandwidthQualityScalerUsageHandlerInterface implementation to signal an +// overuse or underuse of bandwidth (which indicate a desire to scale the video +// stream down or up). +class BandwidthQualityScaler { + public: + explicit BandwidthQualityScaler( + BandwidthQualityScalerUsageHandlerInterface* handler); + virtual ~BandwidthQualityScaler(); + + void ReportEncodeInfo(int frame_size_bytes, + int64_t time_sent_in_ms, + uint32_t encoded_width, + uint32_t encoded_height); + + // We prioritise to using the |resolution_bitrate_limits| provided by the + // current decoder. If not provided, we will use the default data by + // GetDefaultResolutionBitrateLimits(). + void SetResolutionBitrateLimits( + const std::vector<VideoEncoder::ResolutionBitrateLimits>& + resolution_bitrate_limits); + + const TimeDelta kBitrateStateUpdateInterval; + + private: + enum class CheckBitrateResult { + kInsufficientSamples, + kNormalBitrate, + kHighBitRate, + kLowBitRate, + }; + + // We will periodically check encode bitrate, this function will make + // resolution up or down decisions and report the decision to the adapter. + void StartCheckForBitrate(); + CheckBitrateResult CheckBitrate(); + + RTC_NO_UNIQUE_ADDRESS SequenceChecker task_checker_; + BandwidthQualityScalerUsageHandlerInterface* const handler_ + RTC_GUARDED_BY(&task_checker_); + + absl::optional<int64_t> last_time_sent_in_ms_ RTC_GUARDED_BY(&task_checker_); + RateStatistics encoded_bitrate_ RTC_GUARDED_BY(&task_checker_); + absl::optional<int> last_frame_size_pixels_ RTC_GUARDED_BY(&task_checker_); + rtc::WeakPtrFactory<BandwidthQualityScaler> weak_ptr_factory_; + + std::vector<VideoEncoder::ResolutionBitrateLimits> resolution_bitrate_limits_; +}; + +} // namespace webrtc +#endif // MODULES_VIDEO_CODING_UTILITY_BANDWIDTH_QUALITY_SCALER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/bandwidth_quality_scaler_unittest.cc b/third_party/libwebrtc/modules/video_coding/utility/bandwidth_quality_scaler_unittest.cc new file mode 100644 index 0000000000..4e2c759707 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/bandwidth_quality_scaler_unittest.cc @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/bandwidth_quality_scaler.h" + +#include <memory> +#include <string> + +#include "api/units/time_delta.h" +#include "rtc_base/checks.h" +#include "rtc_base/event.h" +#include "rtc_base/experiments/encoder_info_settings.h" +#include "rtc_base/task_queue_for_test.h" +#include "rtc_base/time_utils.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { +constexpr int kFramerateFps = 30; +constexpr TimeDelta kDefaultBitrateStateUpdateInterval = TimeDelta::Seconds(5); +constexpr TimeDelta kDefaultEncodeTime = TimeDelta::Seconds(1) / kFramerateFps; + +} // namespace + +class FakeBandwidthQualityScalerHandler + : public BandwidthQualityScalerUsageHandlerInterface { + public: + ~FakeBandwidthQualityScalerHandler() override = default; + void OnReportUsageBandwidthHigh() override { + adapt_down_event_count_++; + event_.Set(); + } + + void OnReportUsageBandwidthLow() override { + adapt_up_event_count_++; + event_.Set(); + } + + rtc::Event event_; + int adapt_up_event_count_ = 0; + int adapt_down_event_count_ = 0; +}; + +class BandwidthQualityScalerUnderTest : public BandwidthQualityScaler { + public: + explicit BandwidthQualityScalerUnderTest( + BandwidthQualityScalerUsageHandlerInterface* handler) + : BandwidthQualityScaler(handler) {} + + int GetBitrateStateUpdateIntervalMs() { + return this->kBitrateStateUpdateInterval.ms() + 200; + } +}; + +class BandwidthQualityScalerTest + : public ::testing::Test, + public ::testing::WithParamInterface<std::string> { + protected: + enum ScaleDirection { + kKeepScaleNormalBandwidth, + kKeepScaleAboveMaxBandwidth, + kKeepScaleUnderMinBandwidth, + }; + + enum FrameType { + kKeyFrame, + kNormalFrame, + kNormalFrame_Overuse, + kNormalFrame_Underuse, + }; + struct FrameConfig { + FrameConfig(int frame_num, + FrameType frame_type, + int actual_width, + int actual_height) + : frame_num(frame_num), + frame_type(frame_type), + actual_width(actual_width), + actual_height(actual_height) {} + + int frame_num; + FrameType frame_type; + int actual_width; + int actual_height; + }; + + BandwidthQualityScalerTest() + : scoped_field_trial_(GetParam()), + task_queue_("BandwidthQualityScalerTestQueue"), + handler_(std::make_unique<FakeBandwidthQualityScalerHandler>()) { + task_queue_.SendTask([this] { + bandwidth_quality_scaler_ = + std::unique_ptr<BandwidthQualityScalerUnderTest>( + new BandwidthQualityScalerUnderTest(handler_.get())); + bandwidth_quality_scaler_->SetResolutionBitrateLimits( + EncoderInfoSettings:: + GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted()); + // Only for testing. Set first_timestamp_ in RateStatistics to 0. + bandwidth_quality_scaler_->ReportEncodeInfo(0, 0, 0, 0); + }); + } + + ~BandwidthQualityScalerTest() { + task_queue_.SendTask([this] { bandwidth_quality_scaler_ = nullptr; }); + } + + int GetFrameSizeBytes( + const FrameConfig& config, + const VideoEncoder::ResolutionBitrateLimits& bitrate_limits) { + int scale = 8 * kFramerateFps; + switch (config.frame_type) { + case FrameType::kKeyFrame: { + // 4 is experimental value. Based on the test, the number of bytes of + // the key frame is about four times of the normal frame + return bitrate_limits.max_bitrate_bps * 4 / scale; + } + case FrameType::kNormalFrame_Overuse: { + return bitrate_limits.max_bitrate_bps * 3 / 2 / scale; + } + case FrameType::kNormalFrame_Underuse: { + return bitrate_limits.min_start_bitrate_bps * 3 / 4 / scale; + } + case FrameType::kNormalFrame: { + return (bitrate_limits.max_bitrate_bps + + bitrate_limits.min_start_bitrate_bps) / + 2 / scale; + } + } + return -1; + } + + absl::optional<VideoEncoder::ResolutionBitrateLimits> + GetDefaultSuitableBitrateLimit(int frame_size_pixels) { + return EncoderInfoSettings:: + GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted( + frame_size_pixels, + EncoderInfoSettings:: + GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted()); + } + + void TriggerBandwidthQualityScalerTest( + const std::vector<FrameConfig>& frame_configs) { + task_queue_.SendTask([frame_configs, this] { + RTC_CHECK(!frame_configs.empty()); + + int total_frame_nums = 0; + for (const FrameConfig& frame_config : frame_configs) { + total_frame_nums += frame_config.frame_num; + } + + EXPECT_EQ(kFramerateFps * kDefaultBitrateStateUpdateInterval.seconds(), + total_frame_nums); + + uint32_t time_send_to_scaler_ms_ = rtc::TimeMillis(); + for (size_t i = 0; i < frame_configs.size(); ++i) { + const FrameConfig& config = frame_configs[i]; + absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate = + GetDefaultSuitableBitrateLimit(config.actual_width * + config.actual_height); + EXPECT_TRUE(suitable_bitrate); + for (int j = 0; j <= config.frame_num; ++j) { + time_send_to_scaler_ms_ += kDefaultEncodeTime.ms(); + int frame_size_bytes = + GetFrameSizeBytes(config, suitable_bitrate.value()); + RTC_CHECK(frame_size_bytes > 0); + bandwidth_quality_scaler_->ReportEncodeInfo( + frame_size_bytes, time_send_to_scaler_ms_, config.actual_width, + config.actual_height); + } + } + }); + } + + test::ScopedFieldTrials scoped_field_trial_; + TaskQueueForTest task_queue_; + std::unique_ptr<BandwidthQualityScalerUnderTest> bandwidth_quality_scaler_; + std::unique_ptr<FakeBandwidthQualityScalerHandler> handler_; +}; + +INSTANTIATE_TEST_SUITE_P( + FieldTrials, + BandwidthQualityScalerTest, + ::testing::Values("WebRTC-Video-BandwidthQualityScalerSettings/" + "bitrate_state_update_interval_s_:1/", + "WebRTC-Video-BandwidthQualityScalerSettings/" + "bitrate_state_update_interval_s_:2/")); + +TEST_P(BandwidthQualityScalerTest, AllNormalFrame_640x360) { + const std::vector<FrameConfig> frame_configs{ + FrameConfig(150, FrameType::kNormalFrame, 640, 360)}; + TriggerBandwidthQualityScalerTest(frame_configs); + + // When resolution is 640*360, experimental working bitrate range is + // [500000,800000] bps. Encoded bitrate is 654253, so it falls in the range + // without any operation(up/down). + EXPECT_FALSE(handler_->event_.Wait(TimeDelta::Millis( + bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs()))); + EXPECT_EQ(0, handler_->adapt_down_event_count_); + EXPECT_EQ(0, handler_->adapt_up_event_count_); +} + +TEST_P(BandwidthQualityScalerTest, AllNoramlFrame_AboveMaxBandwidth_640x360) { + const std::vector<FrameConfig> frame_configs{ + FrameConfig(150, FrameType::kNormalFrame_Overuse, 640, 360)}; + TriggerBandwidthQualityScalerTest(frame_configs); + + // When resolution is 640*360, experimental working bitrate range is + // [500000,800000] bps. Encoded bitrate is 1208000 > 800000 * 0.95, so it + // triggers adapt_up_event_count_. + EXPECT_TRUE(handler_->event_.Wait(TimeDelta::Millis( + bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs()))); + EXPECT_EQ(0, handler_->adapt_down_event_count_); + EXPECT_EQ(1, handler_->adapt_up_event_count_); +} + +TEST_P(BandwidthQualityScalerTest, AllNormalFrame_Underuse_640x360) { + const std::vector<FrameConfig> frame_configs{ + FrameConfig(150, FrameType::kNormalFrame_Underuse, 640, 360)}; + TriggerBandwidthQualityScalerTest(frame_configs); + + // When resolution is 640*360, experimental working bitrate range is + // [500000,800000] bps. Encoded bitrate is 377379 < 500000 * 0.8, so it + // triggers adapt_down_event_count_. + EXPECT_TRUE(handler_->event_.Wait(TimeDelta::Millis( + bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs()))); + EXPECT_EQ(1, handler_->adapt_down_event_count_); + EXPECT_EQ(0, handler_->adapt_up_event_count_); +} + +TEST_P(BandwidthQualityScalerTest, FixedFrameTypeTest1_640x360) { + const std::vector<FrameConfig> frame_configs{ + FrameConfig(5, FrameType::kNormalFrame_Underuse, 640, 360), + FrameConfig(110, FrameType::kNormalFrame, 640, 360), + FrameConfig(20, FrameType::kNormalFrame_Overuse, 640, 360), + FrameConfig(15, FrameType::kKeyFrame, 640, 360), + }; + TriggerBandwidthQualityScalerTest(frame_configs); + + // When resolution is 640*360, experimental working bitrate range is + // [500000,800000] bps. Encoded bitrate is 1059462 > 800000 * 0.95, so it + // triggers adapt_up_event_count_. + EXPECT_TRUE(handler_->event_.Wait(TimeDelta::Millis( + bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs()))); + EXPECT_EQ(0, handler_->adapt_down_event_count_); + EXPECT_EQ(1, handler_->adapt_up_event_count_); +} + +TEST_P(BandwidthQualityScalerTest, FixedFrameTypeTest2_640x360) { + const std::vector<FrameConfig> frame_configs{ + FrameConfig(10, FrameType::kNormalFrame_Underuse, 640, 360), + FrameConfig(50, FrameType::kNormalFrame, 640, 360), + FrameConfig(5, FrameType::kKeyFrame, 640, 360), + FrameConfig(85, FrameType::kNormalFrame_Overuse, 640, 360), + }; + TriggerBandwidthQualityScalerTest(frame_configs); + + // When resolution is 640*360, experimental working bitrate range is + // [500000,800000] bps. Encoded bitrate is 1059462 > 800000 * 0.95, so it + // triggers adapt_up_event_count_. + EXPECT_TRUE(handler_->event_.Wait(TimeDelta::Millis( + bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs()))); + EXPECT_EQ(0, handler_->adapt_down_event_count_); + EXPECT_EQ(1, handler_->adapt_up_event_count_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/decoded_frames_history.cc b/third_party/libwebrtc/modules/video_coding/utility/decoded_frames_history.cc new file mode 100644 index 0000000000..1138aa8448 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/decoded_frames_history.cc @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/decoded_frames_history.h" + +#include <algorithm> + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace video_coding { + +DecodedFramesHistory::DecodedFramesHistory(size_t window_size) + : buffer_(window_size) {} + +DecodedFramesHistory::~DecodedFramesHistory() = default; + +void DecodedFramesHistory::InsertDecoded(int64_t frame_id, uint32_t timestamp) { + last_decoded_frame_ = frame_id; + last_decoded_frame_timestamp_ = timestamp; + int new_index = FrameIdToIndex(frame_id); + + RTC_DCHECK(last_frame_id_ < frame_id); + + // Clears expired values from the cyclic buffer_. + if (last_frame_id_) { + int64_t id_jump = frame_id - *last_frame_id_; + int last_index = FrameIdToIndex(*last_frame_id_); + + if (id_jump >= static_cast<int64_t>(buffer_.size())) { + std::fill(buffer_.begin(), buffer_.end(), false); + } else if (new_index > last_index) { + std::fill(buffer_.begin() + last_index + 1, buffer_.begin() + new_index, + false); + } else { + std::fill(buffer_.begin() + last_index + 1, buffer_.end(), false); + std::fill(buffer_.begin(), buffer_.begin() + new_index, false); + } + } + + buffer_[new_index] = true; + last_frame_id_ = frame_id; +} + +bool DecodedFramesHistory::WasDecoded(int64_t frame_id) const { + if (!last_frame_id_) + return false; + + // Reference to the picture_id out of the stored should happen. + if (frame_id <= *last_frame_id_ - static_cast<int64_t>(buffer_.size())) { + RTC_LOG(LS_WARNING) << "Referencing a frame out of the window. " + "Assuming it was undecoded to avoid artifacts."; + return false; + } + + if (frame_id > last_frame_id_) + return false; + + return buffer_[FrameIdToIndex(frame_id)]; +} + +void DecodedFramesHistory::Clear() { + last_decoded_frame_timestamp_.reset(); + last_decoded_frame_.reset(); + std::fill(buffer_.begin(), buffer_.end(), false); + last_frame_id_.reset(); +} + +absl::optional<int64_t> DecodedFramesHistory::GetLastDecodedFrameId() const { + return last_decoded_frame_; +} + +absl::optional<uint32_t> DecodedFramesHistory::GetLastDecodedFrameTimestamp() + const { + return last_decoded_frame_timestamp_; +} + +int DecodedFramesHistory::FrameIdToIndex(int64_t frame_id) const { + int m = frame_id % buffer_.size(); + return m >= 0 ? m : m + buffer_.size(); +} + +} // namespace video_coding +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/decoded_frames_history.h b/third_party/libwebrtc/modules/video_coding/utility/decoded_frames_history.h new file mode 100644 index 0000000000..9b8bf65821 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/decoded_frames_history.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_DECODED_FRAMES_HISTORY_H_ +#define MODULES_VIDEO_CODING_UTILITY_DECODED_FRAMES_HISTORY_H_ + +#include <stdint.h> + +#include <bitset> +#include <vector> + +#include "absl/types/optional.h" +#include "api/video/encoded_frame.h" + +namespace webrtc { +namespace video_coding { + +class DecodedFramesHistory { + public: + // window_size - how much frames back to the past are actually remembered. + explicit DecodedFramesHistory(size_t window_size); + ~DecodedFramesHistory(); + // Called for each decoded frame. Assumes frame id's are non-decreasing. + void InsertDecoded(int64_t frame_id, uint32_t timestamp); + // Query if the following (frame_id, spatial_id) pair was inserted before. + // Should be at most less by window_size-1 than the last inserted frame id. + bool WasDecoded(int64_t frame_id) const; + + void Clear(); + + absl::optional<int64_t> GetLastDecodedFrameId() const; + absl::optional<uint32_t> GetLastDecodedFrameTimestamp() const; + + private: + int FrameIdToIndex(int64_t frame_id) const; + + std::vector<bool> buffer_; + absl::optional<int64_t> last_frame_id_; + absl::optional<int64_t> last_decoded_frame_; + absl::optional<uint32_t> last_decoded_frame_timestamp_; +}; + +} // namespace video_coding +} // namespace webrtc +#endif // MODULES_VIDEO_CODING_UTILITY_DECODED_FRAMES_HISTORY_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/decoded_frames_history_unittest.cc b/third_party/libwebrtc/modules/video_coding/utility/decoded_frames_history_unittest.cc new file mode 100644 index 0000000000..ac09a42053 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/decoded_frames_history_unittest.cc @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/decoded_frames_history.h" + +#include "test/gtest.h" + +namespace webrtc { +namespace video_coding { +namespace { + +constexpr int kHistorySize = 1 << 13; + +TEST(DecodedFramesHistory, RequestOnEmptyHistory) { + DecodedFramesHistory history(kHistorySize); + EXPECT_EQ(history.WasDecoded(1234), false); +} + +TEST(DecodedFramesHistory, FindsLastDecodedFrame) { + DecodedFramesHistory history(kHistorySize); + history.InsertDecoded(1234, 0); + EXPECT_EQ(history.WasDecoded(1234), true); +} + +TEST(DecodedFramesHistory, FindsPreviousFrame) { + DecodedFramesHistory history(kHistorySize); + history.InsertDecoded(1234, 0); + history.InsertDecoded(1235, 0); + EXPECT_EQ(history.WasDecoded(1234), true); +} + +TEST(DecodedFramesHistory, ReportsMissingFrame) { + DecodedFramesHistory history(kHistorySize); + history.InsertDecoded(1234, 0); + history.InsertDecoded(1236, 0); + EXPECT_EQ(history.WasDecoded(1235), false); +} + +TEST(DecodedFramesHistory, ClearsHistory) { + DecodedFramesHistory history(kHistorySize); + history.InsertDecoded(1234, 0); + history.Clear(); + EXPECT_EQ(history.WasDecoded(1234), false); + EXPECT_EQ(history.GetLastDecodedFrameId(), absl::nullopt); + EXPECT_EQ(history.GetLastDecodedFrameTimestamp(), absl::nullopt); +} + +TEST(DecodedFramesHistory, HandlesBigJumpInPictureId) { + DecodedFramesHistory history(kHistorySize); + history.InsertDecoded(1234, 0); + history.InsertDecoded(1235, 0); + history.InsertDecoded(1236, 0); + history.InsertDecoded(1236 + kHistorySize / 2, 0); + EXPECT_EQ(history.WasDecoded(1234), true); + EXPECT_EQ(history.WasDecoded(1237), false); +} + +TEST(DecodedFramesHistory, ForgetsTooOldHistory) { + DecodedFramesHistory history(kHistorySize); + history.InsertDecoded(1234, 0); + history.InsertDecoded(1235, 0); + history.InsertDecoded(1236, 0); + history.InsertDecoded(1236 + kHistorySize * 2, 0); + EXPECT_EQ(history.WasDecoded(1234), false); + EXPECT_EQ(history.WasDecoded(1237), false); +} + +TEST(DecodedFramesHistory, ReturnsLastDecodedFrameId) { + DecodedFramesHistory history(kHistorySize); + EXPECT_EQ(history.GetLastDecodedFrameId(), absl::nullopt); + history.InsertDecoded(1234, 0); + EXPECT_EQ(history.GetLastDecodedFrameId(), 1234); + history.InsertDecoded(1235, 0); + EXPECT_EQ(history.GetLastDecodedFrameId(), 1235); +} + +TEST(DecodedFramesHistory, ReturnsLastDecodedFrameTimestamp) { + DecodedFramesHistory history(kHistorySize); + EXPECT_EQ(history.GetLastDecodedFrameTimestamp(), absl::nullopt); + history.InsertDecoded(1234, 12345); + EXPECT_EQ(history.GetLastDecodedFrameTimestamp(), 12345u); + history.InsertDecoded(1235, 12366); + EXPECT_EQ(history.GetLastDecodedFrameTimestamp(), 12366u); +} + +TEST(DecodedFramesHistory, NegativePictureIds) { + DecodedFramesHistory history(kHistorySize); + history.InsertDecoded(-1234, 12345); + history.InsertDecoded(-1233, 12366); + EXPECT_EQ(*history.GetLastDecodedFrameId(), -1233); + + history.InsertDecoded(-1, 12377); + history.InsertDecoded(0, 12388); + EXPECT_EQ(*history.GetLastDecodedFrameId(), 0); + + history.InsertDecoded(1, 12399); + EXPECT_EQ(*history.GetLastDecodedFrameId(), 1); + + EXPECT_EQ(history.WasDecoded(-1234), true); + EXPECT_EQ(history.WasDecoded(-1), true); + EXPECT_EQ(history.WasDecoded(0), true); + EXPECT_EQ(history.WasDecoded(1), true); +} + +} // namespace +} // namespace video_coding +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/frame_dropper.cc b/third_party/libwebrtc/modules/video_coding/utility/frame_dropper.cc new file mode 100644 index 0000000000..8ea8a8e268 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/frame_dropper.cc @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/frame_dropper.h" + +#include <algorithm> + +namespace webrtc { + +namespace { + +const float kDefaultFrameSizeAlpha = 0.9f; +const float kDefaultKeyFrameRatioAlpha = 0.99f; +// 1 key frame every 10th second in 30 fps. +const float kDefaultKeyFrameRatioValue = 1 / 300.0f; + +const float kDefaultDropRatioAlpha = 0.9f; +const float kDefaultDropRatioValue = 0.96f; +// Maximum duration over which frames are continuously dropped. +const float kDefaultMaxDropDurationSecs = 4.0f; + +// Default target bitrate. +// TODO(isheriff): Should this be higher to avoid dropping too many packets when +// the bandwidth is unknown at the start ? +const float kDefaultTargetBitrateKbps = 300.0f; +const float kDefaultIncomingFrameRate = 30; +const float kLeakyBucketSizeSeconds = 0.5f; + +// A delta frame that is bigger than `kLargeDeltaFactor` times the average +// delta frame is a large frame that is spread out for accumulation. +const int kLargeDeltaFactor = 3; + +// Cap on the frame size accumulator to prevent excessive drops. +const float kAccumulatorCapBufferSizeSecs = 3.0f; +} // namespace + +FrameDropper::FrameDropper() + : key_frame_ratio_(kDefaultKeyFrameRatioAlpha), + delta_frame_size_avg_kbits_(kDefaultFrameSizeAlpha), + drop_ratio_(kDefaultDropRatioAlpha, kDefaultDropRatioValue), + enabled_(true), + max_drop_duration_secs_(kDefaultMaxDropDurationSecs) { + Reset(); +} + +FrameDropper::~FrameDropper() = default; + +void FrameDropper::Reset() { + key_frame_ratio_.Reset(kDefaultKeyFrameRatioAlpha); + key_frame_ratio_.Apply(1.0f, kDefaultKeyFrameRatioValue); + delta_frame_size_avg_kbits_.Reset(kDefaultFrameSizeAlpha); + + accumulator_ = 0.0f; + accumulator_max_ = kDefaultTargetBitrateKbps / 2; + target_bitrate_ = kDefaultTargetBitrateKbps; + incoming_frame_rate_ = kDefaultIncomingFrameRate; + + large_frame_accumulation_count_ = 0; + large_frame_accumulation_chunk_size_ = 0; + large_frame_accumulation_spread_ = 0.5 * kDefaultIncomingFrameRate; + + drop_next_ = false; + drop_ratio_.Reset(0.9f); + drop_ratio_.Apply(0.0f, 0.0f); + drop_count_ = 0; + was_below_max_ = true; +} + +void FrameDropper::Enable(bool enable) { + enabled_ = enable; +} + +void FrameDropper::Fill(size_t framesize_bytes, bool delta_frame) { + if (!enabled_) { + return; + } + float framesize_kbits = 8.0f * static_cast<float>(framesize_bytes) / 1000.0f; + if (!delta_frame) { + key_frame_ratio_.Apply(1.0, 1.0); + // Do not spread if we are already doing it (or we risk dropping bits that + // need accumulation). Given we compute the key frame ratio and spread + // based on that, this should not normally happen. + if (large_frame_accumulation_count_ == 0) { + if (key_frame_ratio_.filtered() > 1e-5 && + 1 / key_frame_ratio_.filtered() < large_frame_accumulation_spread_) { + large_frame_accumulation_count_ = + static_cast<int32_t>(1 / key_frame_ratio_.filtered() + 0.5); + } else { + large_frame_accumulation_count_ = + static_cast<int32_t>(large_frame_accumulation_spread_ + 0.5); + } + large_frame_accumulation_chunk_size_ = + framesize_kbits / large_frame_accumulation_count_; + framesize_kbits = 0; + } + } else { + // Identify if it is an unusually large delta frame and spread accumulation + // if that is the case. + if (delta_frame_size_avg_kbits_.filtered() != -1 && + (framesize_kbits > + kLargeDeltaFactor * delta_frame_size_avg_kbits_.filtered()) && + large_frame_accumulation_count_ == 0) { + large_frame_accumulation_count_ = + static_cast<int32_t>(large_frame_accumulation_spread_ + 0.5); + large_frame_accumulation_chunk_size_ = + framesize_kbits / large_frame_accumulation_count_; + framesize_kbits = 0; + } else { + delta_frame_size_avg_kbits_.Apply(1, framesize_kbits); + } + key_frame_ratio_.Apply(1.0, 0.0); + } + // Change the level of the accumulator (bucket) + accumulator_ += framesize_kbits; + CapAccumulator(); +} + +void FrameDropper::Leak(uint32_t input_framerate) { + if (!enabled_) { + return; + } + if (input_framerate < 1) { + return; + } + if (target_bitrate_ < 0.0f) { + return; + } + // Add lower bound for large frame accumulation spread. + large_frame_accumulation_spread_ = std::max(0.5 * input_framerate, 5.0); + // Expected bits per frame based on current input frame rate. + float expected_bits_per_frame = target_bitrate_ / input_framerate; + if (large_frame_accumulation_count_ > 0) { + expected_bits_per_frame -= large_frame_accumulation_chunk_size_; + --large_frame_accumulation_count_; + } + accumulator_ -= expected_bits_per_frame; + if (accumulator_ < 0.0f) { + accumulator_ = 0.0f; + } + UpdateRatio(); +} + +void FrameDropper::UpdateRatio() { + if (accumulator_ > 1.3f * accumulator_max_) { + // Too far above accumulator max, react faster. + drop_ratio_.UpdateBase(0.8f); + } else { + // Go back to normal reaction. + drop_ratio_.UpdateBase(0.9f); + } + if (accumulator_ > accumulator_max_) { + // We are above accumulator max, and should ideally drop a frame. Increase + // the drop_ratio_ and drop the frame later. + if (was_below_max_) { + drop_next_ = true; + } + drop_ratio_.Apply(1.0f, 1.0f); + drop_ratio_.UpdateBase(0.9f); + } else { + drop_ratio_.Apply(1.0f, 0.0f); + } + was_below_max_ = accumulator_ < accumulator_max_; +} + +// This function signals when to drop frames to the caller. It makes use of the +// drop_ratio_ to smooth out the drops over time. +bool FrameDropper::DropFrame() { + if (!enabled_) { + return false; + } + if (drop_next_) { + drop_next_ = false; + drop_count_ = 0; + } + + if (drop_ratio_.filtered() >= 0.5f) { // Drops per keep + // Limit is the number of frames we should drop between each kept frame + // to keep our drop ratio. limit is positive in this case. + float denom = 1.0f - drop_ratio_.filtered(); + if (denom < 1e-5) { + denom = 1e-5f; + } + int32_t limit = static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f); + // Put a bound on the max amount of dropped frames between each kept + // frame, in terms of frame rate and window size (secs). + int max_limit = + static_cast<int>(incoming_frame_rate_ * max_drop_duration_secs_); + if (limit > max_limit) { + limit = max_limit; + } + if (drop_count_ < 0) { + // Reset the drop_count_ since it was negative and should be positive. + drop_count_ = -drop_count_; + } + if (drop_count_ < limit) { + // As long we are below the limit we should drop frames. + drop_count_++; + return true; + } else { + // Only when we reset drop_count_ a frame should be kept. + drop_count_ = 0; + return false; + } + } else if (drop_ratio_.filtered() > 0.0f && + drop_ratio_.filtered() < 0.5f) { // Keeps per drop + // Limit is the number of frames we should keep between each drop + // in order to keep the drop ratio. limit is negative in this case, + // and the drop_count_ is also negative. + float denom = drop_ratio_.filtered(); + if (denom < 1e-5) { + denom = 1e-5f; + } + int32_t limit = -static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f); + if (drop_count_ > 0) { + // Reset the drop_count_ since we have a positive + // drop_count_, and it should be negative. + drop_count_ = -drop_count_; + } + if (drop_count_ > limit) { + if (drop_count_ == 0) { + // Drop frames when we reset drop_count_. + drop_count_--; + return true; + } else { + // Keep frames as long as we haven't reached limit. + drop_count_--; + return false; + } + } else { + drop_count_ = 0; + return false; + } + } + drop_count_ = 0; + return false; +} + +void FrameDropper::SetRates(float bitrate, float incoming_frame_rate) { + // Bit rate of -1 means infinite bandwidth. + accumulator_max_ = bitrate * kLeakyBucketSizeSeconds; + if (target_bitrate_ > 0.0f && bitrate < target_bitrate_ && + accumulator_ > accumulator_max_) { + // Rescale the accumulator level if the accumulator max decreases + accumulator_ = bitrate / target_bitrate_ * accumulator_; + } + target_bitrate_ = bitrate; + CapAccumulator(); + incoming_frame_rate_ = incoming_frame_rate; +} + +// Put a cap on the accumulator, i.e., don't let it grow beyond some level. +// This is a temporary fix for screencasting where very large frames from +// encoder will cause very slow response (too many frame drops). +// TODO(isheriff): Remove this now that large delta frames are also spread out ? +void FrameDropper::CapAccumulator() { + float max_accumulator = target_bitrate_ * kAccumulatorCapBufferSizeSecs; + if (accumulator_ > max_accumulator) { + accumulator_ = max_accumulator; + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/frame_dropper.h b/third_party/libwebrtc/modules/video_coding/utility/frame_dropper.h new file mode 100644 index 0000000000..b45b7fe27f --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/frame_dropper.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_FRAME_DROPPER_H_ +#define MODULES_VIDEO_CODING_UTILITY_FRAME_DROPPER_H_ + +#include <stddef.h> +#include <stdint.h> + +#include "rtc_base/numerics/exp_filter.h" + +namespace webrtc { + +// The Frame Dropper implements a variant of the leaky bucket algorithm +// for keeping track of when to drop frames to avoid bit rate +// over use when the encoder can't keep its bit rate. +class FrameDropper { + public: + FrameDropper(); + ~FrameDropper(); + + // Resets the FrameDropper to its initial state. + void Reset(); + + void Enable(bool enable); + + // Answers the question if it's time to drop a frame if we want to reach a + // given frame rate. Must be called for every frame. + // + // Return value : True if we should drop the current frame. + bool DropFrame(); + + // Updates the FrameDropper with the size of the latest encoded frame. + // The FrameDropper calculates a new drop ratio (can be seen as the + // probability to drop a frame) and updates its internal statistics. + // + // Input: + // - framesize_bytes : The size of the latest frame returned + // from the encoder. + // - delta_frame : True if the encoder returned a delta frame. + void Fill(size_t framesize_bytes, bool delta_frame); + + void Leak(uint32_t input_framerate); + + // Sets the target bit rate and the frame rate produced by the camera. + // + // Input: + // - bitrate : The target bit rate. + void SetRates(float bitrate, float incoming_frame_rate); + + private: + void UpdateRatio(); + void CapAccumulator(); + + rtc::ExpFilter key_frame_ratio_; + rtc::ExpFilter delta_frame_size_avg_kbits_; + + // Key frames and large delta frames are not immediately accumulated in the + // bucket since they can immediately overflow the bucket leading to large + // drops on the following packets that may be much smaller. Instead these + // large frames are accumulated over several frames when the bucket leaks. + + // `large_frame_accumulation_spread_` represents the number of frames over + // which a large frame is accumulated. + float large_frame_accumulation_spread_; + // `large_frame_accumulation_count_` represents the number of frames left + // to finish accumulating a large frame. + int large_frame_accumulation_count_; + // `large_frame_accumulation_chunk_size_` represents the size of a single + // chunk for large frame accumulation. + float large_frame_accumulation_chunk_size_; + + float accumulator_; + float accumulator_max_; + float target_bitrate_; + bool drop_next_; + rtc::ExpFilter drop_ratio_; + int drop_count_; + float incoming_frame_rate_; + bool was_below_max_; + bool enabled_; + const float max_drop_duration_secs_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_UTILITY_FRAME_DROPPER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/frame_dropper_unittest.cc b/third_party/libwebrtc/modules/video_coding/utility/frame_dropper_unittest.cc new file mode 100644 index 0000000000..066103a788 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/frame_dropper_unittest.cc @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/frame_dropper.h" + +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +const float kTargetBitRateKbps = 300; +const float kIncomingFrameRate = 30; +const size_t kFrameSizeBytes = 1250; + +const size_t kLargeFrameSizeBytes = 25000; + +const bool kIncludeKeyFrame = true; +const bool kDoNotIncludeKeyFrame = false; + +} // namespace + +class FrameDropperTest : public ::testing::Test { + protected: + void SetUp() override { + frame_dropper_.SetRates(kTargetBitRateKbps, kIncomingFrameRate); + } + + void OverflowLeakyBucket() { + // Overflow bucket in frame dropper. + for (int i = 0; i < kIncomingFrameRate; ++i) { + frame_dropper_.Fill(kFrameSizeBytes, true); + } + frame_dropper_.Leak(kIncomingFrameRate); + } + + void ValidateNoDropsAtTargetBitrate(int large_frame_size_bytes, + int large_frame_rate, + bool is_large_frame_delta) { + // Smaller frame size is computed to meet `kTargetBitRateKbps`. + int small_frame_size_bytes = + kFrameSizeBytes - + (large_frame_size_bytes * large_frame_rate) / kIncomingFrameRate; + + for (int i = 1; i <= 5 * large_frame_rate; ++i) { + // Large frame. First frame is always a key frame. + frame_dropper_.Fill(large_frame_size_bytes, + (i == 1) ? false : is_large_frame_delta); + frame_dropper_.Leak(kIncomingFrameRate); + EXPECT_FALSE(frame_dropper_.DropFrame()); + + // Smaller frames. + for (int j = 1; j < kIncomingFrameRate / large_frame_rate; ++j) { + frame_dropper_.Fill(small_frame_size_bytes, true); + frame_dropper_.Leak(kIncomingFrameRate); + EXPECT_FALSE(frame_dropper_.DropFrame()); + } + } + } + + void ValidateThroughputMatchesTargetBitrate(int bitrate_kbps, + bool include_keyframe) { + int delta_frame_size; + int total_bytes = 0; + + if (include_keyframe) { + delta_frame_size = ((1000.0 / 8 * bitrate_kbps) - kLargeFrameSizeBytes) / + (kIncomingFrameRate - 1); + } else { + delta_frame_size = bitrate_kbps * 1000.0 / (8 * kIncomingFrameRate); + } + const int kNumIterations = 1000; + for (int i = 1; i <= kNumIterations; ++i) { + int j = 0; + if (include_keyframe) { + if (!frame_dropper_.DropFrame()) { + frame_dropper_.Fill(kLargeFrameSizeBytes, false); + total_bytes += kLargeFrameSizeBytes; + } + frame_dropper_.Leak(kIncomingFrameRate); + j++; + } + for (; j < kIncomingFrameRate; ++j) { + if (!frame_dropper_.DropFrame()) { + frame_dropper_.Fill(delta_frame_size, true); + total_bytes += delta_frame_size; + } + frame_dropper_.Leak(kIncomingFrameRate); + } + } + float throughput_kbps = total_bytes * 8.0 / (1000 * kNumIterations); + float deviation_from_target = + (throughput_kbps - kTargetBitRateKbps) * 100.0 / kTargetBitRateKbps; + if (deviation_from_target < 0) { + deviation_from_target = -deviation_from_target; + } + + // Variation is < 0.1% + EXPECT_LE(deviation_from_target, 0.1); + } + + FrameDropper frame_dropper_; +}; + +TEST_F(FrameDropperTest, NoDropsWhenDisabled) { + frame_dropper_.Enable(false); + OverflowLeakyBucket(); + EXPECT_FALSE(frame_dropper_.DropFrame()); +} + +TEST_F(FrameDropperTest, DropsByDefaultWhenBucketOverflows) { + OverflowLeakyBucket(); + EXPECT_TRUE(frame_dropper_.DropFrame()); +} + +TEST_F(FrameDropperTest, NoDropsWhenFillRateMatchesLeakRate) { + for (int i = 0; i < 5 * kIncomingFrameRate; ++i) { + frame_dropper_.Fill(kFrameSizeBytes, true); + frame_dropper_.Leak(kIncomingFrameRate); + EXPECT_FALSE(frame_dropper_.DropFrame()); + } +} + +TEST_F(FrameDropperTest, LargeKeyFrames) { + ValidateNoDropsAtTargetBitrate(kLargeFrameSizeBytes, 1, false); + frame_dropper_.Reset(); + ValidateNoDropsAtTargetBitrate(kLargeFrameSizeBytes / 2, 2, false); + frame_dropper_.Reset(); + ValidateNoDropsAtTargetBitrate(kLargeFrameSizeBytes / 4, 4, false); + frame_dropper_.Reset(); + ValidateNoDropsAtTargetBitrate(kLargeFrameSizeBytes / 8, 8, false); +} + +TEST_F(FrameDropperTest, LargeDeltaFrames) { + ValidateNoDropsAtTargetBitrate(kLargeFrameSizeBytes, 1, true); + frame_dropper_.Reset(); + ValidateNoDropsAtTargetBitrate(kLargeFrameSizeBytes / 2, 2, true); + frame_dropper_.Reset(); + ValidateNoDropsAtTargetBitrate(kLargeFrameSizeBytes / 4, 4, true); + frame_dropper_.Reset(); + ValidateNoDropsAtTargetBitrate(kLargeFrameSizeBytes / 8, 8, true); +} + +TEST_F(FrameDropperTest, TrafficVolumeAboveAvailableBandwidth) { + ValidateThroughputMatchesTargetBitrate(700, kIncludeKeyFrame); + ValidateThroughputMatchesTargetBitrate(700, kDoNotIncludeKeyFrame); + ValidateThroughputMatchesTargetBitrate(600, kIncludeKeyFrame); + ValidateThroughputMatchesTargetBitrate(600, kDoNotIncludeKeyFrame); + ValidateThroughputMatchesTargetBitrate(500, kIncludeKeyFrame); + ValidateThroughputMatchesTargetBitrate(500, kDoNotIncludeKeyFrame); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/framerate_controller_deprecated.cc b/third_party/libwebrtc/modules/video_coding/utility/framerate_controller_deprecated.cc new file mode 100644 index 0000000000..5978adc3c4 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/framerate_controller_deprecated.cc @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/framerate_controller_deprecated.h" + +#include <stddef.h> + +#include <cstdint> + +namespace webrtc { + +FramerateControllerDeprecated::FramerateControllerDeprecated( + float target_framerate_fps) + : min_frame_interval_ms_(0), framerate_estimator_(1000.0, 1000.0) { + SetTargetRate(target_framerate_fps); +} + +void FramerateControllerDeprecated::SetTargetRate(float target_framerate_fps) { + if (target_framerate_fps_ != target_framerate_fps) { + framerate_estimator_.Reset(); + if (last_timestamp_ms_) { + framerate_estimator_.Update(1, *last_timestamp_ms_); + } + + const size_t target_frame_interval_ms = 1000 / target_framerate_fps; + target_framerate_fps_ = target_framerate_fps; + min_frame_interval_ms_ = 85 * target_frame_interval_ms / 100; + } +} + +float FramerateControllerDeprecated::GetTargetRate() { + return *target_framerate_fps_; +} + +void FramerateControllerDeprecated::Reset() { + framerate_estimator_.Reset(); + last_timestamp_ms_.reset(); +} + +bool FramerateControllerDeprecated::DropFrame(uint32_t timestamp_ms) const { + if (timestamp_ms < last_timestamp_ms_) { + // Timestamp jumps backward. We can't make adequate drop decision. Don't + // drop this frame. Stats will be reset in AddFrame(). + return false; + } + + if (Rate(timestamp_ms).value_or(*target_framerate_fps_) > + target_framerate_fps_) { + return true; + } + + if (last_timestamp_ms_) { + const int64_t diff_ms = + static_cast<int64_t>(timestamp_ms) - *last_timestamp_ms_; + if (diff_ms < min_frame_interval_ms_) { + return true; + } + } + + return false; +} + +void FramerateControllerDeprecated::AddFrame(uint32_t timestamp_ms) { + if (timestamp_ms < last_timestamp_ms_) { + // Timestamp jumps backward. + Reset(); + } + + framerate_estimator_.Update(1, timestamp_ms); + last_timestamp_ms_ = timestamp_ms; +} + +absl::optional<float> FramerateControllerDeprecated::Rate( + uint32_t timestamp_ms) const { + return framerate_estimator_.Rate(timestamp_ms); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/framerate_controller_deprecated.h b/third_party/libwebrtc/modules/video_coding/utility/framerate_controller_deprecated.h new file mode 100644 index 0000000000..ca0cbea053 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/framerate_controller_deprecated.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_FRAMERATE_CONTROLLER_DEPRECATED_H_ +#define MODULES_VIDEO_CODING_UTILITY_FRAMERATE_CONTROLLER_DEPRECATED_H_ + +#include <stdint.h> + +#include "absl/types/optional.h" +#include "rtc_base/rate_statistics.h" + +namespace webrtc { + +// Please use webrtc::FramerateController instead. +class FramerateControllerDeprecated { + public: + explicit FramerateControllerDeprecated(float target_framerate_fps); + + void SetTargetRate(float target_framerate_fps); + float GetTargetRate(); + + // Advices user to drop next frame in order to reach target framerate. + bool DropFrame(uint32_t timestamp_ms) const; + + void AddFrame(uint32_t timestamp_ms); + + void Reset(); + + private: + absl::optional<float> Rate(uint32_t timestamp_ms) const; + + absl::optional<float> target_framerate_fps_; + absl::optional<uint32_t> last_timestamp_ms_; + uint32_t min_frame_interval_ms_; + RateStatistics framerate_estimator_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_UTILITY_FRAMERATE_CONTROLLER_DEPRECATED_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/framerate_controller_deprecated_unittest.cc b/third_party/libwebrtc/modules/video_coding/utility/framerate_controller_deprecated_unittest.cc new file mode 100644 index 0000000000..eabf0529db --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/framerate_controller_deprecated_unittest.cc @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/framerate_controller_deprecated.h" + +#include <stddef.h> + +#include "test/gtest.h" + +namespace webrtc { + +TEST(FramerateControllerDeprecated, KeepTargetFramerate) { + const float input_framerate_fps = 20; + const float target_framerate_fps = 5; + const float max_abs_framerate_error_fps = target_framerate_fps * 0.1f; + const size_t input_duration_secs = 3; + const size_t num_input_frames = input_duration_secs * input_framerate_fps; + + FramerateControllerDeprecated framerate_controller(target_framerate_fps); + size_t num_dropped_frames = 0; + for (size_t frame_num = 0; frame_num < num_input_frames; ++frame_num) { + const uint32_t timestamp_ms = + static_cast<uint32_t>(1000 * frame_num / input_framerate_fps); + if (framerate_controller.DropFrame(timestamp_ms)) { + ++num_dropped_frames; + } else { + framerate_controller.AddFrame(timestamp_ms); + } + } + + const float output_framerate_fps = + static_cast<float>(num_input_frames - num_dropped_frames) / + input_duration_secs; + EXPECT_NEAR(output_framerate_fps, target_framerate_fps, + max_abs_framerate_error_fps); +} + +TEST(FramerateControllerDeprecated, DoNotDropAnyFramesIfTargerEqualsInput) { + const float input_framerate_fps = 30; + const size_t input_duration_secs = 3; + const size_t num_input_frames = input_duration_secs * input_framerate_fps; + + FramerateControllerDeprecated framerate_controller(input_framerate_fps); + size_t num_dropped_frames = 0; + for (size_t frame_num = 0; frame_num < num_input_frames; ++frame_num) { + const uint32_t timestamp_ms = + static_cast<uint32_t>(1000 * frame_num / input_framerate_fps); + if (framerate_controller.DropFrame(timestamp_ms)) { + ++num_dropped_frames; + } else { + framerate_controller.AddFrame(timestamp_ms); + } + } + + EXPECT_EQ(num_dropped_frames, 0U); +} + +TEST(FramerateControllerDeprecated, DoNotDropFrameWhenTimestampJumpsBackward) { + FramerateControllerDeprecated framerate_controller(30); + ASSERT_FALSE(framerate_controller.DropFrame(66)); + framerate_controller.AddFrame(66); + EXPECT_FALSE(framerate_controller.DropFrame(33)); +} + +TEST(FramerateControllerDeprecated, DropFrameIfItIsTooCloseToPreviousFrame) { + FramerateControllerDeprecated framerate_controller(30); + ASSERT_FALSE(framerate_controller.DropFrame(33)); + framerate_controller.AddFrame(33); + EXPECT_TRUE(framerate_controller.DropFrame(34)); +} + +TEST(FramerateControllerDeprecated, FrameDroppingStartsFromSecondInputFrame) { + const float input_framerate_fps = 23; + const float target_framerate_fps = 19; + const uint32_t input_frame_duration_ms = + static_cast<uint32_t>(1000 / input_framerate_fps); + FramerateControllerDeprecated framerate_controller(target_framerate_fps); + ASSERT_FALSE(framerate_controller.DropFrame(1 * input_frame_duration_ms)); + framerate_controller.AddFrame(1 * input_frame_duration_ms); + EXPECT_TRUE(framerate_controller.DropFrame(2 * input_frame_duration_ms)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/ivf_defines.h b/third_party/libwebrtc/modules/video_coding/utility/ivf_defines.h new file mode 100644 index 0000000000..212d381e70 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/ivf_defines.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * This file contains definitions that are common to the IvfFileReader and + * IvfFileWriter classes. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_IVF_DEFINES_H_ +#define MODULES_VIDEO_CODING_UTILITY_IVF_DEFINES_H_ + +#include <stddef.h> + +namespace webrtc { +constexpr size_t kIvfHeaderSize = 32; +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_UTILITY_IVF_DEFINES_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/ivf_file_reader.cc b/third_party/libwebrtc/modules/video_coding/utility/ivf_file_reader.cc new file mode 100644 index 0000000000..4c08ca613a --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/ivf_file_reader.cc @@ -0,0 +1,242 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/ivf_file_reader.h" + +#include <string> +#include <vector> + +#include "api/video_codecs/video_codec.h" +#include "modules/rtp_rtcp/source/byte_io.h" +#include "modules/video_coding/utility/ivf_defines.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr size_t kIvfFrameHeaderSize = 12; +constexpr int kCodecTypeBytesCount = 4; + +constexpr uint8_t kFileHeaderStart[kCodecTypeBytesCount] = {'D', 'K', 'I', 'F'}; +constexpr uint8_t kVp8Header[kCodecTypeBytesCount] = {'V', 'P', '8', '0'}; +constexpr uint8_t kVp9Header[kCodecTypeBytesCount] = {'V', 'P', '9', '0'}; +constexpr uint8_t kAv1Header[kCodecTypeBytesCount] = {'A', 'V', '0', '1'}; +constexpr uint8_t kH264Header[kCodecTypeBytesCount] = {'H', '2', '6', '4'}; +constexpr uint8_t kH265Header[kCodecTypeBytesCount] = {'H', '2', '6', '5'}; + +// RTP standard required 90kHz clock rate. +constexpr int32_t kRtpClockRateHz = 90000; + +} // namespace + +std::unique_ptr<IvfFileReader> IvfFileReader::Create(FileWrapper file) { + auto reader = + std::unique_ptr<IvfFileReader>(new IvfFileReader(std::move(file))); + if (!reader->Reset()) { + return nullptr; + } + return reader; +} +IvfFileReader::~IvfFileReader() { + Close(); +} + +bool IvfFileReader::Reset() { + // Set error to true while initialization. + has_error_ = true; + if (!file_.Rewind()) { + RTC_LOG(LS_ERROR) << "Failed to rewind IVF file"; + return false; + } + + uint8_t ivf_header[kIvfHeaderSize] = {0}; + size_t read = file_.Read(&ivf_header, kIvfHeaderSize); + if (read != kIvfHeaderSize) { + RTC_LOG(LS_ERROR) << "Failed to read IVF header"; + return false; + } + + if (memcmp(&ivf_header[0], kFileHeaderStart, 4) != 0) { + RTC_LOG(LS_ERROR) << "File is not in IVF format: DKIF header expected"; + return false; + } + + absl::optional<VideoCodecType> codec_type = ParseCodecType(ivf_header, 8); + if (!codec_type) { + return false; + } + codec_type_ = *codec_type; + + width_ = ByteReader<uint16_t>::ReadLittleEndian(&ivf_header[12]); + height_ = ByteReader<uint16_t>::ReadLittleEndian(&ivf_header[14]); + if (width_ == 0 || height_ == 0) { + RTC_LOG(LS_ERROR) << "Invalid IVF header: width or height is 0"; + return false; + } + + time_scale_ = ByteReader<uint32_t>::ReadLittleEndian(&ivf_header[16]); + if (time_scale_ == 0) { + RTC_LOG(LS_ERROR) << "Invalid IVF header: time scale can't be 0"; + return false; + } + + num_frames_ = static_cast<size_t>( + ByteReader<uint32_t>::ReadLittleEndian(&ivf_header[24])); + if (num_frames_ <= 0) { + RTC_LOG(LS_ERROR) << "Invalid IVF header: number of frames 0 or negative"; + return false; + } + + num_read_frames_ = 0; + next_frame_header_ = ReadNextFrameHeader(); + if (!next_frame_header_) { + RTC_LOG(LS_ERROR) << "Failed to read 1st frame header"; + return false; + } + // Initialization succeed: reset error. + has_error_ = false; + + const char* codec_name = CodecTypeToPayloadString(codec_type_); + RTC_LOG(LS_INFO) << "Opened IVF file with codec data of type " << codec_name + << " at resolution " << width_ << " x " << height_ + << ", using " << time_scale_ << "Hz clock resolution."; + + return true; +} + +absl::optional<EncodedImage> IvfFileReader::NextFrame() { + if (has_error_ || !HasMoreFrames()) { + return absl::nullopt; + } + + rtc::scoped_refptr<EncodedImageBuffer> payload = EncodedImageBuffer::Create(); + std::vector<size_t> layer_sizes; + // next_frame_header_ have to be presented by the way how it was loaded. If it + // is missing it means there is a bug in error handling. + RTC_DCHECK(next_frame_header_); + int64_t current_timestamp = next_frame_header_->timestamp; + // The first frame from the file should be marked as Key frame. + bool is_first_frame = num_read_frames_ == 0; + while (next_frame_header_ && + current_timestamp == next_frame_header_->timestamp) { + // Resize payload to fit next spatial layer. + size_t current_layer_size = next_frame_header_->frame_size; + size_t current_layer_start_pos = payload->size(); + payload->Realloc(payload->size() + current_layer_size); + layer_sizes.push_back(current_layer_size); + + // Read next layer into payload + size_t read = file_.Read(&payload->data()[current_layer_start_pos], + current_layer_size); + if (read != current_layer_size) { + RTC_LOG(LS_ERROR) << "Frame #" << num_read_frames_ + << ": failed to read frame payload"; + has_error_ = true; + return absl::nullopt; + } + num_read_frames_++; + + current_timestamp = next_frame_header_->timestamp; + next_frame_header_ = ReadNextFrameHeader(); + } + if (!next_frame_header_) { + // If EOF was reached, we need to check that all frames were met. + if (!has_error_ && num_read_frames_ != num_frames_) { + RTC_LOG(LS_ERROR) << "Unexpected EOF"; + has_error_ = true; + return absl::nullopt; + } + } + + EncodedImage image; + image.capture_time_ms_ = current_timestamp; + image.SetRtpTimestamp( + static_cast<uint32_t>(current_timestamp * kRtpClockRateHz / time_scale_)); + image.SetEncodedData(payload); + image.SetSpatialIndex(static_cast<int>(layer_sizes.size()) - 1); + for (size_t i = 0; i < layer_sizes.size(); ++i) { + image.SetSpatialLayerFrameSize(static_cast<int>(i), layer_sizes[i]); + } + if (is_first_frame) { + image._frameType = VideoFrameType::kVideoFrameKey; + } + + return image; +} + +bool IvfFileReader::Close() { + if (!file_.is_open()) + return false; + + file_.Close(); + return true; +} + +absl::optional<VideoCodecType> IvfFileReader::ParseCodecType(uint8_t* buffer, + size_t start_pos) { + if (memcmp(&buffer[start_pos], kVp8Header, kCodecTypeBytesCount) == 0) { + return VideoCodecType::kVideoCodecVP8; + } + if (memcmp(&buffer[start_pos], kVp9Header, kCodecTypeBytesCount) == 0) { + return VideoCodecType::kVideoCodecVP9; + } + if (memcmp(&buffer[start_pos], kAv1Header, kCodecTypeBytesCount) == 0) { + return VideoCodecType::kVideoCodecAV1; + } + if (memcmp(&buffer[start_pos], kH264Header, kCodecTypeBytesCount) == 0) { + return VideoCodecType::kVideoCodecH264; + } + if (memcmp(&buffer[start_pos], kH265Header, kCodecTypeBytesCount) == 0) { + return VideoCodecType::kVideoCodecH265; + } + has_error_ = true; + RTC_LOG(LS_ERROR) << "Unknown codec type: " + << std::string( + reinterpret_cast<char const*>(&buffer[start_pos]), + kCodecTypeBytesCount); + return absl::nullopt; +} + +absl::optional<IvfFileReader::FrameHeader> +IvfFileReader::ReadNextFrameHeader() { + uint8_t ivf_frame_header[kIvfFrameHeaderSize] = {0}; + size_t read = file_.Read(&ivf_frame_header, kIvfFrameHeaderSize); + if (read != kIvfFrameHeaderSize) { + if (read != 0 || !file_.ReadEof()) { + has_error_ = true; + RTC_LOG(LS_ERROR) << "Frame #" << num_read_frames_ + << ": failed to read IVF frame header"; + } + return absl::nullopt; + } + FrameHeader header; + header.frame_size = static_cast<size_t>( + ByteReader<uint32_t>::ReadLittleEndian(&ivf_frame_header[0])); + header.timestamp = + ByteReader<uint64_t>::ReadLittleEndian(&ivf_frame_header[4]); + + if (header.frame_size == 0) { + has_error_ = true; + RTC_LOG(LS_ERROR) << "Frame #" << num_read_frames_ + << ": invalid frame size"; + return absl::nullopt; + } + + if (header.timestamp < 0) { + has_error_ = true; + RTC_LOG(LS_ERROR) << "Frame #" << num_read_frames_ + << ": negative timestamp"; + return absl::nullopt; + } + + return header; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/ivf_file_reader.h b/third_party/libwebrtc/modules/video_coding/utility/ivf_file_reader.h new file mode 100644 index 0000000000..db4fc25575 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/ivf_file_reader.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_IVF_FILE_READER_H_ +#define MODULES_VIDEO_CODING_UTILITY_IVF_FILE_READER_H_ + +#include <memory> +#include <utility> + +#include "absl/types/optional.h" +#include "api/video/encoded_image.h" +#include "api/video_codecs/video_codec.h" +#include "rtc_base/system/file_wrapper.h" + +namespace webrtc { + +class IvfFileReader { + public: + // Creates IvfFileReader. Returns nullptr if error acquired. + static std::unique_ptr<IvfFileReader> Create(FileWrapper file); + ~IvfFileReader(); + + IvfFileReader(const IvfFileReader&) = delete; + IvfFileReader& operator=(const IvfFileReader&) = delete; + + // Reinitializes reader. Returns false if any error acquired. + bool Reset(); + + // Returns codec type which was used to create this IVF file and which should + // be used to decode EncodedImages from this file. + VideoCodecType GetVideoCodecType() const { return codec_type_; } + // Returns count of frames in this file. + size_t GetFramesCount() const { return num_frames_; } + + // Returns next frame or absl::nullopt if any error acquired. Always returns + // absl::nullopt after first error was spotted. + absl::optional<EncodedImage> NextFrame(); + bool HasMoreFrames() const { return num_read_frames_ < num_frames_; } + bool HasError() const { return has_error_; } + + uint16_t GetFrameWidth() const { return width_; } + uint16_t GetFrameHeight() const { return height_; } + + bool Close(); + + private: + struct FrameHeader { + size_t frame_size; + int64_t timestamp; + }; + + explicit IvfFileReader(FileWrapper file) : file_(std::move(file)) {} + + // Parses codec type from specified position of the buffer. Codec type + // contains kCodecTypeBytesCount bytes and caller has to ensure that buffer + // won't overflow. + absl::optional<VideoCodecType> ParseCodecType(uint8_t* buffer, + size_t start_pos); + absl::optional<FrameHeader> ReadNextFrameHeader(); + + VideoCodecType codec_type_; + size_t num_frames_; + size_t num_read_frames_; + uint16_t width_; + uint16_t height_; + uint32_t time_scale_; + FileWrapper file_; + + absl::optional<FrameHeader> next_frame_header_; + bool has_error_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_UTILITY_IVF_FILE_READER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/ivf_file_reader_unittest.cc b/third_party/libwebrtc/modules/video_coding/utility/ivf_file_reader_unittest.cc new file mode 100644 index 0000000000..14bfdcae14 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/ivf_file_reader_unittest.cc @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/ivf_file_reader.h" + +#include <memory> +#include <string> + +#include "modules/video_coding/utility/ivf_file_writer.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace { + +constexpr int kWidth = 320; +constexpr int kHeight = 240; +constexpr int kNumFrames = 3; +constexpr uint8_t kDummyPayload[4] = {'0', '1', '2', '3'}; + +} // namespace + +class IvfFileReaderTest : public ::testing::Test { + protected: + void SetUp() override { + file_name_ = + webrtc::test::TempFilename(webrtc::test::OutputPath(), "test_file.ivf"); + } + void TearDown() override { webrtc::test::RemoveFile(file_name_); } + + bool WriteDummyTestFrames(IvfFileWriter* file_writer, + VideoCodecType codec_type, + int width, + int height, + int num_frames, + bool use_capture_tims_ms, + int spatial_layers_count) { + EncodedImage frame; + frame.SetSpatialIndex(spatial_layers_count); + rtc::scoped_refptr<EncodedImageBuffer> payload = EncodedImageBuffer::Create( + sizeof(kDummyPayload) * spatial_layers_count); + for (int i = 0; i < spatial_layers_count; ++i) { + memcpy(&payload->data()[i * sizeof(kDummyPayload)], kDummyPayload, + sizeof(kDummyPayload)); + frame.SetSpatialLayerFrameSize(i, sizeof(kDummyPayload)); + } + frame.SetEncodedData(payload); + frame._encodedWidth = width; + frame._encodedHeight = height; + for (int i = 1; i <= num_frames; ++i) { + if (use_capture_tims_ms) { + frame.capture_time_ms_ = i; + } else { + frame.SetRtpTimestamp(i); + } + if (!file_writer->WriteFrame(frame, codec_type)) + return false; + } + return true; + } + + void CreateTestFile(VideoCodecType codec_type, + bool use_capture_tims_ms, + int spatial_layers_count) { + std::unique_ptr<IvfFileWriter> file_writer = + IvfFileWriter::Wrap(FileWrapper::OpenWriteOnly(file_name_), 0); + ASSERT_TRUE(file_writer.get()); + ASSERT_TRUE(WriteDummyTestFrames(file_writer.get(), codec_type, kWidth, + kHeight, kNumFrames, use_capture_tims_ms, + spatial_layers_count)); + ASSERT_TRUE(file_writer->Close()); + } + + void ValidateFrame(absl::optional<EncodedImage> frame, + int frame_index, + bool use_capture_tims_ms, + int spatial_layers_count) { + ASSERT_TRUE(frame); + EXPECT_EQ(frame->SpatialIndex(), spatial_layers_count - 1); + if (use_capture_tims_ms) { + EXPECT_EQ(frame->capture_time_ms_, static_cast<int64_t>(frame_index)); + EXPECT_EQ(frame->RtpTimestamp(), static_cast<int64_t>(90 * frame_index)); + } else { + EXPECT_EQ(frame->RtpTimestamp(), static_cast<int64_t>(frame_index)); + } + ASSERT_EQ(frame->size(), sizeof(kDummyPayload) * spatial_layers_count); + for (int i = 0; i < spatial_layers_count; ++i) { + EXPECT_EQ(memcmp(&frame->data()[i * sizeof(kDummyPayload)], kDummyPayload, + sizeof(kDummyPayload)), + 0) + << std::string(reinterpret_cast<char const*>( + &frame->data()[i * sizeof(kDummyPayload)]), + sizeof(kDummyPayload)); + } + } + + void ValidateContent(VideoCodecType codec_type, + bool use_capture_tims_ms, + int spatial_layers_count) { + std::unique_ptr<IvfFileReader> reader = + IvfFileReader::Create(FileWrapper::OpenReadOnly(file_name_)); + ASSERT_TRUE(reader.get()); + EXPECT_EQ(reader->GetVideoCodecType(), codec_type); + EXPECT_EQ(reader->GetFramesCount(), + spatial_layers_count * static_cast<size_t>(kNumFrames)); + for (int i = 1; i <= kNumFrames; ++i) { + ASSERT_TRUE(reader->HasMoreFrames()); + ValidateFrame(reader->NextFrame(), i, use_capture_tims_ms, + spatial_layers_count); + EXPECT_FALSE(reader->HasError()); + } + EXPECT_FALSE(reader->HasMoreFrames()); + EXPECT_FALSE(reader->NextFrame()); + EXPECT_FALSE(reader->HasError()); + ASSERT_TRUE(reader->Close()); + } + + std::string file_name_; +}; + +TEST_F(IvfFileReaderTest, BasicVp8FileNtpTimestamp) { + CreateTestFile(kVideoCodecVP8, false, 1); + ValidateContent(kVideoCodecVP8, false, 1); +} + +TEST_F(IvfFileReaderTest, BasicVP8FileMsTimestamp) { + CreateTestFile(kVideoCodecVP8, true, 1); + ValidateContent(kVideoCodecVP8, true, 1); +} + +TEST_F(IvfFileReaderTest, BasicVP9FileNtpTimestamp) { + CreateTestFile(kVideoCodecVP9, false, 1); + ValidateContent(kVideoCodecVP9, false, 1); +} + +TEST_F(IvfFileReaderTest, BasicVP9FileMsTimestamp) { + CreateTestFile(kVideoCodecVP9, true, 1); + ValidateContent(kVideoCodecVP9, true, 1); +} + +TEST_F(IvfFileReaderTest, BasicAv1FileNtpTimestamp) { + CreateTestFile(kVideoCodecAV1, false, 1); + ValidateContent(kVideoCodecAV1, false, 1); +} + +TEST_F(IvfFileReaderTest, BasicAv1FileMsTimestamp) { + CreateTestFile(kVideoCodecAV1, true, 1); + ValidateContent(kVideoCodecAV1, true, 1); +} + +TEST_F(IvfFileReaderTest, BasicH264FileNtpTimestamp) { + CreateTestFile(kVideoCodecH264, false, 1); + ValidateContent(kVideoCodecH264, false, 1); +} + +TEST_F(IvfFileReaderTest, BasicH264FileMsTimestamp) { + CreateTestFile(kVideoCodecH264, true, 1); + ValidateContent(kVideoCodecH264, true, 1); +} + +TEST_F(IvfFileReaderTest, MultilayerVp8FileNtpTimestamp) { + CreateTestFile(kVideoCodecVP8, false, 3); + ValidateContent(kVideoCodecVP8, false, 3); +} + +TEST_F(IvfFileReaderTest, MultilayerVP9FileNtpTimestamp) { + CreateTestFile(kVideoCodecVP9, false, 3); + ValidateContent(kVideoCodecVP9, false, 3); +} + +TEST_F(IvfFileReaderTest, MultilayerAv1FileNtpTimestamp) { + CreateTestFile(kVideoCodecAV1, false, 3); + ValidateContent(kVideoCodecAV1, false, 3); +} + +TEST_F(IvfFileReaderTest, MultilayerH264FileNtpTimestamp) { + CreateTestFile(kVideoCodecH264, false, 3); + ValidateContent(kVideoCodecH264, false, 3); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/ivf_file_writer.cc b/third_party/libwebrtc/modules/video_coding/utility/ivf_file_writer.cc new file mode 100644 index 0000000000..422e4585a8 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/ivf_file_writer.cc @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/ivf_file_writer.h" + +#include <cstddef> +#include <cstdint> +#include <memory> +#include <utility> + +#include "absl/strings/string_view.h" +#include "api/video/encoded_image.h" +#include "api/video/video_codec_type.h" +#include "api/video_codecs/video_codec.h" +#include "modules/rtp_rtcp/source/byte_io.h" +#include "modules/video_coding/utility/ivf_defines.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/system/file_wrapper.h" + +// TODO(palmkvist): make logging more informative in the absence of a file name +// (or get one) + +namespace webrtc { + +namespace { + +constexpr int kDefaultWidth = 1280; +constexpr int kDefaultHeight = 720; +} // namespace + +IvfFileWriter::IvfFileWriter(FileWrapper file, size_t byte_limit) + : codec_type_(kVideoCodecGeneric), + bytes_written_(0), + byte_limit_(byte_limit), + num_frames_(0), + width_(0), + height_(0), + last_timestamp_(-1), + using_capture_timestamps_(false), + file_(std::move(file)) { + RTC_DCHECK(byte_limit == 0 || kIvfHeaderSize <= byte_limit) + << "The byte_limit is too low, not even the header will fit."; +} + +IvfFileWriter::~IvfFileWriter() { + Close(); +} + +std::unique_ptr<IvfFileWriter> IvfFileWriter::Wrap(FileWrapper file, + size_t byte_limit) { + return std::unique_ptr<IvfFileWriter>( + new IvfFileWriter(std::move(file), byte_limit)); +} + +std::unique_ptr<IvfFileWriter> IvfFileWriter::Wrap(absl::string_view filename, + size_t byte_limit) { + return std::unique_ptr<IvfFileWriter>( + new IvfFileWriter(FileWrapper::OpenWriteOnly(filename), byte_limit)); +} + +bool IvfFileWriter::WriteHeader() { + if (!file_.Rewind()) { + RTC_LOG(LS_WARNING) << "Unable to rewind ivf output file."; + return false; + } + + uint8_t ivf_header[kIvfHeaderSize] = {0}; + ivf_header[0] = 'D'; + ivf_header[1] = 'K'; + ivf_header[2] = 'I'; + ivf_header[3] = 'F'; + ByteWriter<uint16_t>::WriteLittleEndian(&ivf_header[4], 0); // Version. + ByteWriter<uint16_t>::WriteLittleEndian(&ivf_header[6], 32); // Header size. + + switch (codec_type_) { + case kVideoCodecVP8: + ivf_header[8] = 'V'; + ivf_header[9] = 'P'; + ivf_header[10] = '8'; + ivf_header[11] = '0'; + break; + case kVideoCodecVP9: + ivf_header[8] = 'V'; + ivf_header[9] = 'P'; + ivf_header[10] = '9'; + ivf_header[11] = '0'; + break; + case kVideoCodecAV1: + ivf_header[8] = 'A'; + ivf_header[9] = 'V'; + ivf_header[10] = '0'; + ivf_header[11] = '1'; + break; + case kVideoCodecH264: + ivf_header[8] = 'H'; + ivf_header[9] = '2'; + ivf_header[10] = '6'; + ivf_header[11] = '4'; + break; + case kVideoCodecH265: + ivf_header[8] = 'H'; + ivf_header[9] = '2'; + ivf_header[10] = '6'; + ivf_header[11] = '5'; + break; + default: + // For unknown codec type use **** code. You can specify actual payload + // format when playing the video with ffplay: ffplay -f H263 file.ivf + ivf_header[8] = '*'; + ivf_header[9] = '*'; + ivf_header[10] = '*'; + ivf_header[11] = '*'; + break; + } + + ByteWriter<uint16_t>::WriteLittleEndian(&ivf_header[12], width_); + ByteWriter<uint16_t>::WriteLittleEndian(&ivf_header[14], height_); + // Render timestamps are in ms (1/1000 scale), while RTP timestamps use a + // 90kHz clock. + ByteWriter<uint32_t>::WriteLittleEndian( + &ivf_header[16], using_capture_timestamps_ ? 1000 : 90000); + ByteWriter<uint32_t>::WriteLittleEndian(&ivf_header[20], 1); + ByteWriter<uint32_t>::WriteLittleEndian(&ivf_header[24], + static_cast<uint32_t>(num_frames_)); + ByteWriter<uint32_t>::WriteLittleEndian(&ivf_header[28], 0); // Reserved. + + if (!file_.Write(ivf_header, kIvfHeaderSize)) { + RTC_LOG(LS_ERROR) << "Unable to write IVF header for ivf output file."; + return false; + } + + if (bytes_written_ < kIvfHeaderSize) { + bytes_written_ = kIvfHeaderSize; + } + + return true; +} + +bool IvfFileWriter::InitFromFirstFrame(const EncodedImage& encoded_image, + VideoCodecType codec_type) { + if (encoded_image._encodedWidth == 0 || encoded_image._encodedHeight == 0) { + width_ = kDefaultWidth; + height_ = kDefaultHeight; + } else { + width_ = encoded_image._encodedWidth; + height_ = encoded_image._encodedHeight; + } + + using_capture_timestamps_ = encoded_image.RtpTimestamp() == 0; + + codec_type_ = codec_type; + + if (!WriteHeader()) + return false; + + const char* codec_name = CodecTypeToPayloadString(codec_type_); + RTC_LOG(LS_WARNING) << "Created IVF file for codec data of type " + << codec_name << " at resolution " << width_ << " x " + << height_ << ", using " + << (using_capture_timestamps_ ? "1" : "90") + << "kHz clock resolution."; + return true; +} + +bool IvfFileWriter::WriteFrame(const EncodedImage& encoded_image, + VideoCodecType codec_type) { + if (!file_.is_open()) + return false; + + if (num_frames_ == 0 && !InitFromFirstFrame(encoded_image, codec_type)) + return false; + RTC_DCHECK_EQ(codec_type_, codec_type); + + int64_t timestamp = using_capture_timestamps_ + ? encoded_image.capture_time_ms_ + : wrap_handler_.Unwrap(encoded_image.RtpTimestamp()); + if (last_timestamp_ != -1 && timestamp < last_timestamp_) { + RTC_LOG(LS_WARNING) << "Timestamp not increasing: " << last_timestamp_ + << " -> " << timestamp; + } + last_timestamp_ = timestamp; + + bool written_frames = false; + size_t max_sl_index = encoded_image.SpatialIndex().value_or(0); + const uint8_t* data = encoded_image.data(); + for (size_t sl_idx = 0; sl_idx <= max_sl_index; ++sl_idx) { + size_t cur_size = encoded_image.SpatialLayerFrameSize(sl_idx).value_or(0); + if (cur_size > 0) { + written_frames = true; + if (!WriteOneSpatialLayer(timestamp, data, cur_size)) { + return false; + } + data += cur_size; + } + } + + // If frame has only one spatial layer it won't have any spatial layers' + // sizes. Therefore this case should be addressed separately. + if (!written_frames) { + return WriteOneSpatialLayer(timestamp, data, encoded_image.size()); + } else { + return true; + } +} + +bool IvfFileWriter::WriteOneSpatialLayer(int64_t timestamp, + const uint8_t* data, + size_t size) { + const size_t kFrameHeaderSize = 12; + if (byte_limit_ != 0 && + bytes_written_ + kFrameHeaderSize + size > byte_limit_) { + RTC_LOG(LS_WARNING) << "Closing IVF file due to reaching size limit: " + << byte_limit_ << " bytes."; + Close(); + return false; + } + uint8_t frame_header[kFrameHeaderSize] = {}; + ByteWriter<uint32_t>::WriteLittleEndian(&frame_header[0], + static_cast<uint32_t>(size)); + ByteWriter<uint64_t>::WriteLittleEndian(&frame_header[4], timestamp); + if (!file_.Write(frame_header, kFrameHeaderSize) || + !file_.Write(data, size)) { + RTC_LOG(LS_ERROR) << "Unable to write frame to file."; + return false; + } + + bytes_written_ += kFrameHeaderSize + size; + + ++num_frames_; + return true; +} + +bool IvfFileWriter::Close() { + if (!file_.is_open()) + return false; + + if (num_frames_ == 0) { + file_.Close(); + return true; + } + + bool ret = WriteHeader(); + file_.Close(); + return ret; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/ivf_file_writer.h b/third_party/libwebrtc/modules/video_coding/utility/ivf_file_writer.h new file mode 100644 index 0000000000..c1c088690b --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/ivf_file_writer.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_IVF_FILE_WRITER_H_ +#define MODULES_VIDEO_CODING_UTILITY_IVF_FILE_WRITER_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <memory> + +#include "absl/strings/string_view.h" +#include "api/video/encoded_image.h" +#include "api/video/video_codec_type.h" +#include "rtc_base/numerics/sequence_number_unwrapper.h" +#include "rtc_base/system/file_wrapper.h" + +namespace webrtc { + +class IvfFileWriter { + public: + // Takes ownership of the file, which will be closed either through + // Close or ~IvfFileWriter. If writing a frame would take the file above the + // `byte_limit` the file will be closed, the write (and all future writes) + // will fail. A `byte_limit` of 0 is equivalent to no limit. + static std::unique_ptr<IvfFileWriter> Wrap(FileWrapper file, + size_t byte_limit); + static std::unique_ptr<IvfFileWriter> Wrap(absl::string_view filename, + size_t byte_limit); + ~IvfFileWriter(); + + IvfFileWriter(const IvfFileWriter&) = delete; + IvfFileWriter& operator=(const IvfFileWriter&) = delete; + + bool WriteFrame(const EncodedImage& encoded_image, VideoCodecType codec_type); + bool Close(); + + private: + explicit IvfFileWriter(FileWrapper file, size_t byte_limit); + + bool WriteHeader(); + bool InitFromFirstFrame(const EncodedImage& encoded_image, + VideoCodecType codec_type); + bool WriteOneSpatialLayer(int64_t timestamp, + const uint8_t* data, + size_t size); + + VideoCodecType codec_type_; + size_t bytes_written_; + size_t byte_limit_; + size_t num_frames_; + uint16_t width_; + uint16_t height_; + int64_t last_timestamp_; + bool using_capture_timestamps_; + RtpTimestampUnwrapper wrap_handler_; + FileWrapper file_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_UTILITY_IVF_FILE_WRITER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/ivf_file_writer_unittest.cc b/third_party/libwebrtc/modules/video_coding/utility/ivf_file_writer_unittest.cc new file mode 100644 index 0000000000..bc6ab65b34 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/ivf_file_writer_unittest.cc @@ -0,0 +1,311 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/ivf_file_writer.h" + +#include <string.h> + +#include <memory> +#include <string> + +#include "modules/rtp_rtcp/source/byte_io.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { + +namespace { +static const int kHeaderSize = 32; +static const int kFrameHeaderSize = 12; +static uint8_t dummy_payload[4] = {0, 1, 2, 3}; +// As the default parameter when the width and height of encodedImage are 0, +// the values are copied from ivf_file_writer.cc +constexpr int kDefaultWidth = 1280; +constexpr int kDefaultHeight = 720; +} // namespace + +class IvfFileWriterTest : public ::testing::Test { + protected: + void SetUp() override { + file_name_ = + webrtc::test::TempFilename(webrtc::test::OutputPath(), "test_file"); + } + void TearDown() override { webrtc::test::RemoveFile(file_name_); } + + bool WriteDummyTestFrames(VideoCodecType codec_type, + int width, + int height, + int num_frames, + bool use_capture_tims_ms) { + EncodedImage frame; + frame.SetEncodedData( + EncodedImageBuffer::Create(dummy_payload, sizeof(dummy_payload))); + frame._encodedWidth = width; + frame._encodedHeight = height; + for (int i = 1; i <= num_frames; ++i) { + frame.set_size(i % sizeof(dummy_payload)); + if (use_capture_tims_ms) { + frame.capture_time_ms_ = i; + } else { + frame.SetRtpTimestamp(i); + } + if (!file_writer_->WriteFrame(frame, codec_type)) + return false; + } + return true; + } + + void VerifyIvfHeader(FileWrapper* file, + const uint8_t fourcc[4], + int width, + int height, + uint32_t num_frames, + bool use_capture_tims_ms) { + ASSERT_TRUE(file->is_open()); + uint8_t data[kHeaderSize]; + ASSERT_EQ(static_cast<size_t>(kHeaderSize), file->Read(data, kHeaderSize)); + + uint8_t dkif[4] = {'D', 'K', 'I', 'F'}; + EXPECT_EQ(0, memcmp(dkif, data, 4)); + EXPECT_EQ(0u, ByteReader<uint16_t>::ReadLittleEndian(&data[4])); + EXPECT_EQ(32u, ByteReader<uint16_t>::ReadLittleEndian(&data[6])); + EXPECT_EQ(0, memcmp(fourcc, &data[8], 4)); + EXPECT_EQ(width, ByteReader<uint16_t>::ReadLittleEndian(&data[12])); + EXPECT_EQ(height, ByteReader<uint16_t>::ReadLittleEndian(&data[14])); + EXPECT_EQ(use_capture_tims_ms ? 1000u : 90000u, + ByteReader<uint32_t>::ReadLittleEndian(&data[16])); + EXPECT_EQ(1u, ByteReader<uint32_t>::ReadLittleEndian(&data[20])); + EXPECT_EQ(num_frames, ByteReader<uint32_t>::ReadLittleEndian(&data[24])); + EXPECT_EQ(0u, ByteReader<uint32_t>::ReadLittleEndian(&data[28])); + } + + void VerifyDummyTestFrames(FileWrapper* file, uint32_t num_frames) { + const int kMaxFrameSize = 4; + for (uint32_t i = 1; i <= num_frames; ++i) { + uint8_t frame_header[kFrameHeaderSize]; + ASSERT_EQ(static_cast<unsigned int>(kFrameHeaderSize), + file->Read(frame_header, kFrameHeaderSize)); + uint32_t frame_length = + ByteReader<uint32_t>::ReadLittleEndian(&frame_header[0]); + EXPECT_EQ(i % 4, frame_length); + uint64_t timestamp = + ByteReader<uint64_t>::ReadLittleEndian(&frame_header[4]); + EXPECT_EQ(i, timestamp); + + uint8_t data[kMaxFrameSize] = {}; + ASSERT_EQ(frame_length, + static_cast<uint32_t>(file->Read(data, frame_length))); + EXPECT_EQ(0, memcmp(data, dummy_payload, frame_length)); + } + } + + void RunBasicFileStructureTest(VideoCodecType codec_type, + const uint8_t fourcc[4], + bool use_capture_tims_ms) { + file_writer_ = + IvfFileWriter::Wrap(FileWrapper::OpenWriteOnly(file_name_), 0); + ASSERT_TRUE(file_writer_.get()); + const int kWidth = 320; + const int kHeight = 240; + const int kNumFrames = 257; + ASSERT_TRUE(WriteDummyTestFrames(codec_type, kWidth, kHeight, kNumFrames, + use_capture_tims_ms)); + EXPECT_TRUE(file_writer_->Close()); + + FileWrapper out_file = FileWrapper::OpenReadOnly(file_name_); + VerifyIvfHeader(&out_file, fourcc, kWidth, kHeight, kNumFrames, + use_capture_tims_ms); + VerifyDummyTestFrames(&out_file, kNumFrames); + + out_file.Close(); + } + + std::string file_name_; + std::unique_ptr<IvfFileWriter> file_writer_; +}; + +TEST_F(IvfFileWriterTest, WritesBasicVP8FileNtpTimestamp) { + const uint8_t fourcc[4] = {'V', 'P', '8', '0'}; + RunBasicFileStructureTest(kVideoCodecVP8, fourcc, false); +} + +TEST_F(IvfFileWriterTest, WritesBasicVP8FileMsTimestamp) { + const uint8_t fourcc[4] = {'V', 'P', '8', '0'}; + RunBasicFileStructureTest(kVideoCodecVP8, fourcc, true); +} + +TEST_F(IvfFileWriterTest, WritesBasicVP9FileNtpTimestamp) { + const uint8_t fourcc[4] = {'V', 'P', '9', '0'}; + RunBasicFileStructureTest(kVideoCodecVP9, fourcc, false); +} + +TEST_F(IvfFileWriterTest, WritesBasicVP9FileMsTimestamp) { + const uint8_t fourcc[4] = {'V', 'P', '9', '0'}; + RunBasicFileStructureTest(kVideoCodecVP9, fourcc, true); +} + +TEST_F(IvfFileWriterTest, WritesBasicAv1FileNtpTimestamp) { + const uint8_t fourcc[4] = {'A', 'V', '0', '1'}; + RunBasicFileStructureTest(kVideoCodecAV1, fourcc, false); +} + +TEST_F(IvfFileWriterTest, WritesBasicAv1FileMsTimestamp) { + const uint8_t fourcc[4] = {'A', 'V', '0', '1'}; + RunBasicFileStructureTest(kVideoCodecAV1, fourcc, true); +} + +TEST_F(IvfFileWriterTest, WritesBasicH264FileNtpTimestamp) { + const uint8_t fourcc[4] = {'H', '2', '6', '4'}; + RunBasicFileStructureTest(kVideoCodecH264, fourcc, false); +} + +TEST_F(IvfFileWriterTest, WritesBasicH264FileMsTimestamp) { + const uint8_t fourcc[4] = {'H', '2', '6', '4'}; + RunBasicFileStructureTest(kVideoCodecH264, fourcc, true); +} + +TEST_F(IvfFileWriterTest, WritesBasicUnknownCodecFileMsTimestamp) { + const uint8_t fourcc[4] = {'*', '*', '*', '*'}; + RunBasicFileStructureTest(kVideoCodecGeneric, fourcc, true); +} + +TEST_F(IvfFileWriterTest, ClosesWhenReachesLimit) { + const uint8_t fourcc[4] = {'V', 'P', '8', '0'}; + const int kWidth = 320; + const int kHeight = 240; + const int kNumFramesToWrite = 2; + const int kNumFramesToFit = 1; + + file_writer_ = IvfFileWriter::Wrap( + FileWrapper::OpenWriteOnly(file_name_), + kHeaderSize + + kNumFramesToFit * (kFrameHeaderSize + sizeof(dummy_payload))); + ASSERT_TRUE(file_writer_.get()); + + ASSERT_FALSE(WriteDummyTestFrames(kVideoCodecVP8, kWidth, kHeight, + kNumFramesToWrite, true)); + ASSERT_FALSE(file_writer_->Close()); + + FileWrapper out_file = FileWrapper::OpenReadOnly(file_name_); + VerifyIvfHeader(&out_file, fourcc, kWidth, kHeight, kNumFramesToFit, true); + VerifyDummyTestFrames(&out_file, kNumFramesToFit); + + out_file.Close(); +} + +TEST_F(IvfFileWriterTest, UseDefaultValueWhenWidthAndHeightAreZero) { + const uint8_t fourcc[4] = {'V', 'P', '8', '0'}; + const int kWidth = 0; + const int kHeight = 0; + const int kNumFramesToWrite = 2; + const int kNumFramesToFit = 1; + + file_writer_ = IvfFileWriter::Wrap( + FileWrapper::OpenWriteOnly(file_name_), + kHeaderSize + + kNumFramesToFit * (kFrameHeaderSize + sizeof(dummy_payload))); + ASSERT_TRUE(file_writer_.get()); + + ASSERT_FALSE(WriteDummyTestFrames(kVideoCodecVP8, kWidth, kHeight, + kNumFramesToWrite, true)); + ASSERT_FALSE(file_writer_->Close()); + + FileWrapper out_file = FileWrapper::OpenReadOnly(file_name_); + // When the width and height are zero, we should expect the width and height + // in IvfHeader to be kDefaultWidth and kDefaultHeight instead of kWidth and + // kHeight. + VerifyIvfHeader(&out_file, fourcc, kDefaultWidth, kDefaultHeight, + kNumFramesToFit, true); + VerifyDummyTestFrames(&out_file, kNumFramesToFit); + + out_file.Close(); +} + +TEST_F(IvfFileWriterTest, UseDefaultValueWhenOnlyWidthIsZero) { + const uint8_t fourcc[4] = {'V', 'P', '8', '0'}; + const int kWidth = 0; + const int kHeight = 360; + const int kNumFramesToWrite = 2; + const int kNumFramesToFit = 1; + + file_writer_ = IvfFileWriter::Wrap( + FileWrapper::OpenWriteOnly(file_name_), + kHeaderSize + + kNumFramesToFit * (kFrameHeaderSize + sizeof(dummy_payload))); + ASSERT_TRUE(file_writer_.get()); + + ASSERT_FALSE(WriteDummyTestFrames(kVideoCodecVP8, kWidth, kHeight, + kNumFramesToWrite, true)); + ASSERT_FALSE(file_writer_->Close()); + + FileWrapper out_file = FileWrapper::OpenReadOnly(file_name_); + // When the width and height are zero, we should expect the width and height + // in IvfHeader to be kDefaultWidth and kDefaultHeight instead of kWidth and + // kHeight. + VerifyIvfHeader(&out_file, fourcc, kDefaultWidth, kDefaultHeight, + kNumFramesToFit, true); + VerifyDummyTestFrames(&out_file, kNumFramesToFit); + + out_file.Close(); +} + +TEST_F(IvfFileWriterTest, UseDefaultValueWhenOnlyHeightIsZero) { + const uint8_t fourcc[4] = {'V', 'P', '8', '0'}; + const int kWidth = 240; + const int kHeight = 0; + const int kNumFramesToWrite = 2; + const int kNumFramesToFit = 1; + + file_writer_ = IvfFileWriter::Wrap( + FileWrapper::OpenWriteOnly(file_name_), + kHeaderSize + + kNumFramesToFit * (kFrameHeaderSize + sizeof(dummy_payload))); + ASSERT_TRUE(file_writer_.get()); + + ASSERT_FALSE(WriteDummyTestFrames(kVideoCodecVP8, kWidth, kHeight, + kNumFramesToWrite, true)); + ASSERT_FALSE(file_writer_->Close()); + + FileWrapper out_file = FileWrapper::OpenReadOnly(file_name_); + // When the width and height are zero, we should expect the width and height + // in IvfHeader to be kDefaultWidth and kDefaultHeight instead of kWidth and + // kHeight. + VerifyIvfHeader(&out_file, fourcc, kDefaultWidth, kDefaultHeight, + kNumFramesToFit, true); + VerifyDummyTestFrames(&out_file, kNumFramesToFit); + + out_file.Close(); +} + +TEST_F(IvfFileWriterTest, UseDefaultValueWhenHeightAndWidthAreNotZero) { + const uint8_t fourcc[4] = {'V', 'P', '8', '0'}; + const int kWidth = 360; + const int kHeight = 240; + const int kNumFramesToWrite = 2; + const int kNumFramesToFit = 1; + + file_writer_ = IvfFileWriter::Wrap( + FileWrapper::OpenWriteOnly(file_name_), + kHeaderSize + + kNumFramesToFit * (kFrameHeaderSize + sizeof(dummy_payload))); + ASSERT_TRUE(file_writer_.get()); + + ASSERT_FALSE(WriteDummyTestFrames(kVideoCodecVP8, kWidth, kHeight, + kNumFramesToWrite, true)); + ASSERT_FALSE(file_writer_->Close()); + + FileWrapper out_file = FileWrapper::OpenReadOnly(file_name_); + VerifyIvfHeader(&out_file, fourcc, kWidth, kHeight, kNumFramesToFit, true); + VerifyDummyTestFrames(&out_file, kNumFramesToFit); + + out_file.Close(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/qp_parser.cc b/third_party/libwebrtc/modules/video_coding/utility/qp_parser.cc new file mode 100644 index 0000000000..3b9aaa377e --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/qp_parser.cc @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/qp_parser.h" + +#include "modules/video_coding/utility/vp8_header_parser.h" +#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" + +namespace webrtc { + +absl::optional<uint32_t> QpParser::Parse(VideoCodecType codec_type, + size_t spatial_idx, + const uint8_t* frame_data, + size_t frame_size) { + if (frame_data == nullptr || frame_size == 0 || + spatial_idx >= kMaxSimulcastStreams) { + return absl::nullopt; + } + + if (codec_type == kVideoCodecVP8) { + int qp = -1; + if (vp8::GetQp(frame_data, frame_size, &qp)) { + return qp; + } + } else if (codec_type == kVideoCodecVP9) { + int qp = -1; + if (vp9::GetQp(frame_data, frame_size, &qp)) { + return qp; + } + } else if (codec_type == kVideoCodecH264) { + return h264_parsers_[spatial_idx].Parse(frame_data, frame_size); + } else if (codec_type == kVideoCodecH265) { + // TODO(bugs.webrtc.org/13485) + } + + return absl::nullopt; +} + +absl::optional<uint32_t> QpParser::H264QpParser::Parse( + const uint8_t* frame_data, + size_t frame_size) { + MutexLock lock(&mutex_); + bitstream_parser_.ParseBitstream( + rtc::ArrayView<const uint8_t>(frame_data, frame_size)); + return bitstream_parser_.GetLastSliceQp(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/qp_parser.h b/third_party/libwebrtc/modules/video_coding/utility/qp_parser.h new file mode 100644 index 0000000000..f132ff9337 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/qp_parser.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_QP_PARSER_H_ +#define MODULES_VIDEO_CODING_UTILITY_QP_PARSER_H_ + +#include "absl/types/optional.h" +#include "api/video/video_codec_constants.h" +#include "api/video/video_codec_type.h" +#include "common_video/h264/h264_bitstream_parser.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +class QpParser { + public: + absl::optional<uint32_t> Parse(VideoCodecType codec_type, + size_t spatial_idx, + const uint8_t* frame_data, + size_t frame_size); + + private: + // A thread safe wrapper for H264 bitstream parser. + class H264QpParser { + public: + absl::optional<uint32_t> Parse(const uint8_t* frame_data, + size_t frame_size); + + private: + Mutex mutex_; + H264BitstreamParser bitstream_parser_ RTC_GUARDED_BY(mutex_); + }; + + H264QpParser h264_parsers_[kMaxSimulcastStreams]; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_UTILITY_QP_PARSER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/qp_parser_unittest.cc b/third_party/libwebrtc/modules/video_coding/utility/qp_parser_unittest.cc new file mode 100644 index 0000000000..1131288f26 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/qp_parser_unittest.cc @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/qp_parser.h" + +#include <stddef.h> + +#include "test/gtest.h" + +namespace webrtc { + +namespace { +// ffmpeg -s 16x16 -f rawvideo -pix_fmt rgb24 -r 30 -i /dev/zero -c:v libvpx +// -qmin 20 -qmax 20 -crf 20 -frames:v 1 -y out.ivf +const uint8_t kCodedFrameVp8Qp25[] = { + 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00, + 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c, + 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0}; + +// ffmpeg -s 16x16 -f rawvideo -pix_fmt rgb24 -r 30 -i /dev/zero -c:v libvpx-vp9 +// -qmin 24 -qmax 24 -crf 24 -frames:v 1 -y out.ivf +const uint8_t kCodedFrameVp9Qp96[] = { + 0xa2, 0x49, 0x83, 0x42, 0xe0, 0x00, 0xf0, 0x00, 0xf6, 0x00, + 0x38, 0x24, 0x1c, 0x18, 0xc0, 0x00, 0x00, 0x30, 0x70, 0x00, + 0x00, 0x4a, 0xa7, 0xff, 0xfc, 0xb9, 0x01, 0xbf, 0xff, 0xff, + 0x97, 0x20, 0xdb, 0xff, 0xff, 0xcb, 0x90, 0x5d, 0x40}; + +// ffmpeg -s 16x16 -f rawvideo -pix_fmt yuv420p -r 30 -i /dev/zero -c:v libx264 +// -qmin 38 -qmax 38 -crf 38 -profile:v baseline -frames:v 2 -y out.264 +const uint8_t kCodedFrameH264SpsPpsIdrQp38[] = { + 0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x0a, 0xd9, 0x1e, 0x84, + 0x00, 0x00, 0x03, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0xf0, 0x3c, + 0x48, 0x99, 0x20, 0x00, 0x00, 0x00, 0x01, 0x68, 0xcb, 0x80, 0xc4, + 0xb2, 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0xf1, 0x18, 0xa0, 0x00, + 0x20, 0x5b, 0x1c, 0x00, 0x04, 0x07, 0xe3, 0x80, 0x00, 0x80, 0xfe}; + +const uint8_t kCodedFrameH264SpsPpsIdrQp49[] = { + 0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x0a, 0xd9, 0x1e, 0x84, + 0x00, 0x00, 0x03, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0xf0, 0x3c, + 0x48, 0x99, 0x20, 0x00, 0x00, 0x00, 0x01, 0x68, 0xcb, 0x80, 0x5d, + 0x2c, 0x80, 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0xf1, 0x18, 0xa0, + 0x00, 0x5e, 0x38, 0x00, 0x08, 0x03, 0xc7, 0x00, 0x01, 0x00, 0x7c}; + +const uint8_t kCodedFrameH264InterSliceQpDelta0[] = {0x00, 0x00, 0x00, 0x01, + 0x41, 0x9a, 0x39, 0xea}; + +} // namespace + +TEST(QpParserTest, ParseQpVp8) { + QpParser parser; + absl::optional<uint32_t> qp = parser.Parse( + kVideoCodecVP8, 0, kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)); + EXPECT_EQ(qp, 25u); +} + +TEST(QpParserTest, ParseQpVp9) { + QpParser parser; + absl::optional<uint32_t> qp = parser.Parse( + kVideoCodecVP9, 0, kCodedFrameVp9Qp96, sizeof(kCodedFrameVp9Qp96)); + EXPECT_EQ(qp, 96u); +} + +TEST(QpParserTest, ParseQpH264) { + QpParser parser; + absl::optional<uint32_t> qp = parser.Parse( + VideoCodecType::kVideoCodecH264, 0, kCodedFrameH264SpsPpsIdrQp38, + sizeof(kCodedFrameH264SpsPpsIdrQp38)); + EXPECT_EQ(qp, 38u); + + qp = parser.Parse(kVideoCodecH264, 1, kCodedFrameH264SpsPpsIdrQp49, + sizeof(kCodedFrameH264SpsPpsIdrQp49)); + EXPECT_EQ(qp, 49u); + + qp = parser.Parse(kVideoCodecH264, 0, kCodedFrameH264InterSliceQpDelta0, + sizeof(kCodedFrameH264InterSliceQpDelta0)); + EXPECT_EQ(qp, 38u); + + qp = parser.Parse(kVideoCodecH264, 1, kCodedFrameH264InterSliceQpDelta0, + sizeof(kCodedFrameH264InterSliceQpDelta0)); + EXPECT_EQ(qp, 49u); +} + +TEST(QpParserTest, ParseQpUnsupportedCodecType) { + QpParser parser; + absl::optional<uint32_t> qp = parser.Parse( + kVideoCodecGeneric, 0, kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)); + EXPECT_FALSE(qp.has_value()); +} + +TEST(QpParserTest, ParseQpNullData) { + QpParser parser; + absl::optional<uint32_t> qp = parser.Parse(kVideoCodecVP8, 0, nullptr, 100); + EXPECT_FALSE(qp.has_value()); +} + +TEST(QpParserTest, ParseQpEmptyData) { + QpParser parser; + absl::optional<uint32_t> qp = + parser.Parse(kVideoCodecVP8, 0, kCodedFrameVp8Qp25, 0); + EXPECT_FALSE(qp.has_value()); +} + +TEST(QpParserTest, ParseQpSpatialIdxExceedsMax) { + QpParser parser; + absl::optional<uint32_t> qp = + parser.Parse(kVideoCodecVP8, kMaxSimulcastStreams, kCodedFrameVp8Qp25, + sizeof(kCodedFrameVp8Qp25)); + EXPECT_FALSE(qp.has_value()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/quality_scaler.cc b/third_party/libwebrtc/modules/video_coding/utility/quality_scaler.cc new file mode 100644 index 0000000000..7ecb340ea4 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/quality_scaler.cc @@ -0,0 +1,336 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/quality_scaler.h" + +#include <memory> +#include <utility> + +#include "api/field_trials_view.h" +#include "api/units/time_delta.h" +#include "api/video/video_adaptation_reason.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/quality_scaler_settings.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/exp_filter.h" +#include "rtc_base/weak_ptr.h" + +namespace webrtc { + +namespace { +// Threshold constant used until first downscale (to permit fast rampup). +static const int kMeasureMs = 2000; +static const float kSamplePeriodScaleFactor = 2.5; +static const int kFramedropPercentThreshold = 60; +static const size_t kMinFramesNeededToScale = 2 * 30; + +} // namespace + +class QualityScaler::QpSmoother { + public: + explicit QpSmoother(float alpha) + : alpha_(alpha), + // The initial value of last_sample_ms doesn't matter since the smoother + // will ignore the time delta for the first update. + last_sample_ms_(0), + smoother_(alpha) {} + + absl::optional<int> GetAvg() const { + float value = smoother_.filtered(); + if (value == rtc::ExpFilter::kValueUndefined) { + return absl::nullopt; + } + return static_cast<int>(value); + } + + void Add(float sample, int64_t time_sent_us) { + int64_t now_ms = time_sent_us / 1000; + smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample); + last_sample_ms_ = now_ms; + } + + void Reset() { smoother_.Reset(alpha_); } + + private: + const float alpha_; + int64_t last_sample_ms_; + rtc::ExpFilter smoother_; +}; + +// The QualityScaler checks for QP periodically by queuing CheckQpTasks. The +// task will either run to completion and trigger a new task being queued, or it +// will be destroyed because the QualityScaler is destroyed. +// +// When high or low QP is reported, the task will be pending until a callback is +// invoked. This lets the QualityScalerQpUsageHandlerInterface react to QP usage +// asynchronously and prevents checking for QP until the stream has potentially +// been reconfigured. +class QualityScaler::CheckQpTask { + public: + // The result of one CheckQpTask may influence the delay of the next + // CheckQpTask. + struct Result { + bool observed_enough_frames = false; + bool qp_usage_reported = false; + }; + + CheckQpTask(QualityScaler* quality_scaler, Result previous_task_result) + : quality_scaler_(quality_scaler), + state_(State::kNotStarted), + previous_task_result_(previous_task_result), + weak_ptr_factory_(this) {} + + void StartDelayedTask() { + RTC_DCHECK_EQ(state_, State::kNotStarted); + state_ = State::kCheckingQp; + TaskQueueBase::Current()->PostDelayedTask( + [this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] { + if (!this_weak_ptr) { + // The task has been cancelled through destruction. + return; + } + RTC_DCHECK_EQ(state_, State::kCheckingQp); + RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_); + switch (quality_scaler_->CheckQp()) { + case QualityScaler::CheckQpResult::kInsufficientSamples: { + result_.observed_enough_frames = false; + // After this line, `this` may be deleted. + break; + } + case QualityScaler::CheckQpResult::kNormalQp: { + result_.observed_enough_frames = true; + break; + } + case QualityScaler::CheckQpResult::kHighQp: { + result_.observed_enough_frames = true; + result_.qp_usage_reported = true; + quality_scaler_->fast_rampup_ = false; + quality_scaler_->handler_->OnReportQpUsageHigh(); + quality_scaler_->ClearSamples(); + break; + } + case QualityScaler::CheckQpResult::kLowQp: { + result_.observed_enough_frames = true; + result_.qp_usage_reported = true; + quality_scaler_->handler_->OnReportQpUsageLow(); + quality_scaler_->ClearSamples(); + break; + } + } + state_ = State::kCompleted; + // Starting the next task deletes the pending task. After this line, + // `this` has been deleted. + quality_scaler_->StartNextCheckQpTask(); + }, + TimeDelta::Millis(GetCheckingQpDelayMs())); + } + + bool HasCompletedTask() const { return state_ == State::kCompleted; } + + Result result() const { + RTC_DCHECK(HasCompletedTask()); + return result_; + } + + private: + enum class State { + kNotStarted, + kCheckingQp, + kCompleted, + }; + + // Determines the sampling period of CheckQpTasks. + int64_t GetCheckingQpDelayMs() const { + RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_); + if (quality_scaler_->fast_rampup_) { + return quality_scaler_->sampling_period_ms_; + } + if (quality_scaler_->experiment_enabled_ && + !previous_task_result_.observed_enough_frames) { + // Use half the interval while waiting for enough frames. + return quality_scaler_->sampling_period_ms_ / 2; + } + if (quality_scaler_->scale_factor_ && + !previous_task_result_.qp_usage_reported) { + // Last CheckQp did not call AdaptDown/Up, possibly reduce interval. + return quality_scaler_->sampling_period_ms_ * + quality_scaler_->scale_factor_.value(); + } + return quality_scaler_->sampling_period_ms_ * + quality_scaler_->initial_scale_factor_; + } + + QualityScaler* const quality_scaler_; + State state_; + const Result previous_task_result_; + Result result_; + + rtc::WeakPtrFactory<CheckQpTask> weak_ptr_factory_; +}; + +QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler, + VideoEncoder::QpThresholds thresholds, + const FieldTrialsView& field_trials) + : QualityScaler(handler, thresholds, field_trials, kMeasureMs) {} + +// Protected ctor, should not be called directly. +QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler, + VideoEncoder::QpThresholds thresholds, + const FieldTrialsView& field_trials, + int64_t default_sampling_period_ms) + : handler_(handler), + thresholds_(thresholds), + sampling_period_ms_(QualityScalerSettings(field_trials) + .SamplingPeriodMs() + .value_or(default_sampling_period_ms)), + fast_rampup_(true), + // Arbitrarily choose size based on 30 fps for 5 seconds. + average_qp_(QualityScalerSettings(field_trials) + .AverageQpWindow() + .value_or(5 * 30)), + framedrop_percent_media_opt_(5 * 30), + framedrop_percent_all_(5 * 30), + experiment_enabled_(QualityScalingExperiment::Enabled(field_trials)), + min_frames_needed_(QualityScalerSettings(field_trials) + .MinFrames() + .value_or(kMinFramesNeededToScale)), + initial_scale_factor_(QualityScalerSettings(field_trials) + .InitialScaleFactor() + .value_or(kSamplePeriodScaleFactor)), + scale_factor_(QualityScalerSettings(field_trials).ScaleFactor()) { + RTC_DCHECK_RUN_ON(&task_checker_); + if (experiment_enabled_) { + config_ = QualityScalingExperiment::GetConfig(field_trials); + qp_smoother_high_.reset(new QpSmoother(config_.alpha_high)); + qp_smoother_low_.reset(new QpSmoother(config_.alpha_low)); + } + RTC_DCHECK(handler_ != nullptr); + StartNextCheckQpTask(); + RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low + << ", high: " << thresholds_.high; +} + +QualityScaler::~QualityScaler() { + RTC_DCHECK_RUN_ON(&task_checker_); +} + +void QualityScaler::StartNextCheckQpTask() { + RTC_DCHECK_RUN_ON(&task_checker_); + RTC_DCHECK(!pending_qp_task_ || pending_qp_task_->HasCompletedTask()) + << "A previous CheckQpTask has not completed yet!"; + CheckQpTask::Result previous_task_result; + if (pending_qp_task_) { + previous_task_result = pending_qp_task_->result(); + } + pending_qp_task_ = std::make_unique<CheckQpTask>(this, previous_task_result); + pending_qp_task_->StartDelayedTask(); +} + +void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) { + RTC_DCHECK_RUN_ON(&task_checker_); + thresholds_ = thresholds; +} + +void QualityScaler::ReportDroppedFrameByMediaOpt() { + RTC_DCHECK_RUN_ON(&task_checker_); + framedrop_percent_media_opt_.AddSample(100); + framedrop_percent_all_.AddSample(100); +} + +void QualityScaler::ReportDroppedFrameByEncoder() { + RTC_DCHECK_RUN_ON(&task_checker_); + framedrop_percent_all_.AddSample(100); +} + +void QualityScaler::ReportQp(int qp, int64_t time_sent_us) { + RTC_DCHECK_RUN_ON(&task_checker_); + framedrop_percent_media_opt_.AddSample(0); + framedrop_percent_all_.AddSample(0); + average_qp_.AddSample(qp); + if (qp_smoother_high_) + qp_smoother_high_->Add(qp, time_sent_us); + if (qp_smoother_low_) + qp_smoother_low_->Add(qp, time_sent_us); +} + +bool QualityScaler::QpFastFilterLow() const { + RTC_DCHECK_RUN_ON(&task_checker_); + size_t num_frames = config_.use_all_drop_reasons + ? framedrop_percent_all_.Size() + : framedrop_percent_media_opt_.Size(); + const size_t kMinNumFrames = 10; + if (num_frames < kMinNumFrames) { + return false; // Wait for more frames before making a decision. + } + absl::optional<int> avg_qp_high = qp_smoother_high_ + ? qp_smoother_high_->GetAvg() + : average_qp_.GetAverageRoundedDown(); + return (avg_qp_high) ? (avg_qp_high.value() <= thresholds_.low) : false; +} + +QualityScaler::CheckQpResult QualityScaler::CheckQp() const { + RTC_DCHECK_RUN_ON(&task_checker_); + // Should be set through InitEncode -> Should be set by now. + RTC_DCHECK_GE(thresholds_.low, 0); + + // If we have not observed at least this many frames we can't make a good + // scaling decision. + const size_t frames = config_.use_all_drop_reasons + ? framedrop_percent_all_.Size() + : framedrop_percent_media_opt_.Size(); + if (frames < min_frames_needed_) { + return CheckQpResult::kInsufficientSamples; + } + + // Check if we should scale down due to high frame drop. + const absl::optional<int> drop_rate = + config_.use_all_drop_reasons + ? framedrop_percent_all_.GetAverageRoundedDown() + : framedrop_percent_media_opt_.GetAverageRoundedDown(); + if (drop_rate && *drop_rate >= kFramedropPercentThreshold) { + RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate; + return CheckQpResult::kHighQp; + } + + // Check if we should scale up or down based on QP. + const absl::optional<int> avg_qp_high = + qp_smoother_high_ ? qp_smoother_high_->GetAvg() + : average_qp_.GetAverageRoundedDown(); + const absl::optional<int> avg_qp_low = + qp_smoother_low_ ? qp_smoother_low_->GetAvg() + : average_qp_.GetAverageRoundedDown(); + if (avg_qp_high && avg_qp_low) { + RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " (" + << *avg_qp_low << ")."; + if (*avg_qp_high > thresholds_.high) { + return CheckQpResult::kHighQp; + } + if (*avg_qp_low <= thresholds_.low) { + // QP has been low. We want to try a higher resolution. + return CheckQpResult::kLowQp; + } + } + return CheckQpResult::kNormalQp; +} + +void QualityScaler::ClearSamples() { + RTC_DCHECK_RUN_ON(&task_checker_); + framedrop_percent_media_opt_.Reset(); + framedrop_percent_all_.Reset(); + average_qp_.Reset(); + if (qp_smoother_high_) + qp_smoother_high_->Reset(); + if (qp_smoother_low_) + qp_smoother_low_->Reset(); +} + +QualityScalerQpUsageHandlerInterface::~QualityScalerQpUsageHandlerInterface() {} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/quality_scaler.h b/third_party/libwebrtc/modules/video_coding/utility/quality_scaler.h new file mode 100644 index 0000000000..14623ff1f3 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/quality_scaler.h @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_ +#define MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <memory> + +#include "absl/types/optional.h" +#include "api/field_trials_view.h" +#include "api/scoped_refptr.h" +#include "api/sequence_checker.h" +#include "api/video_codecs/video_encoder.h" +#include "rtc_base/experiments/quality_scaling_experiment.h" +#include "rtc_base/numerics/moving_average.h" +#include "rtc_base/ref_count.h" +#include "rtc_base/system/no_unique_address.h" + +namespace webrtc { + +class QualityScalerQpUsageHandlerCallbackInterface; +class QualityScalerQpUsageHandlerInterface; + +// QualityScaler runs asynchronously and monitors QP values of encoded frames. +// It holds a reference to a QualityScalerQpUsageHandlerInterface implementation +// to signal an overuse or underuse of QP (which indicate a desire to scale the +// video stream down or up). +class QualityScaler { + public: + // Construct a QualityScaler with given `thresholds` and `handler`. + // This starts the quality scaler periodically checking what the average QP + // has been recently. + QualityScaler(QualityScalerQpUsageHandlerInterface* handler, + VideoEncoder::QpThresholds thresholds, + const FieldTrialsView& field_trials); + virtual ~QualityScaler(); + // Should be called each time a frame is dropped at encoding. + void ReportDroppedFrameByMediaOpt(); + void ReportDroppedFrameByEncoder(); + // Inform the QualityScaler of the last seen QP. + void ReportQp(int qp, int64_t time_sent_us); + + void SetQpThresholds(VideoEncoder::QpThresholds thresholds); + bool QpFastFilterLow() const; + + // The following members declared protected for testing purposes. + protected: + QualityScaler(QualityScalerQpUsageHandlerInterface* handler, + VideoEncoder::QpThresholds thresholds, + const FieldTrialsView& field_trials, + int64_t sampling_period_ms); + + private: + class QpSmoother; + class CheckQpTask; + class CheckQpTaskHandlerCallback; + + enum class CheckQpResult { + kInsufficientSamples, + kNormalQp, + kHighQp, + kLowQp, + }; + + // Starts checking for QP in a delayed task. When the resulting CheckQpTask + // completes, it will invoke this method again, ensuring that we always + // periodically check for QP. See CheckQpTask for more details. We never run + // more than one CheckQpTask at a time. + void StartNextCheckQpTask(); + + CheckQpResult CheckQp() const; + void ClearSamples(); + + std::unique_ptr<CheckQpTask> pending_qp_task_ RTC_GUARDED_BY(&task_checker_); + QualityScalerQpUsageHandlerInterface* const handler_ + RTC_GUARDED_BY(&task_checker_); + RTC_NO_UNIQUE_ADDRESS SequenceChecker task_checker_; + + VideoEncoder::QpThresholds thresholds_ RTC_GUARDED_BY(&task_checker_); + const int64_t sampling_period_ms_; + bool fast_rampup_ RTC_GUARDED_BY(&task_checker_); + rtc::MovingAverage average_qp_ RTC_GUARDED_BY(&task_checker_); + rtc::MovingAverage framedrop_percent_media_opt_ + RTC_GUARDED_BY(&task_checker_); + rtc::MovingAverage framedrop_percent_all_ RTC_GUARDED_BY(&task_checker_); + + // Used by QualityScalingExperiment. + const bool experiment_enabled_; + QualityScalingExperiment::Config config_ RTC_GUARDED_BY(&task_checker_); + std::unique_ptr<QpSmoother> qp_smoother_high_ RTC_GUARDED_BY(&task_checker_); + std::unique_ptr<QpSmoother> qp_smoother_low_ RTC_GUARDED_BY(&task_checker_); + + const size_t min_frames_needed_; + const double initial_scale_factor_; + const absl::optional<double> scale_factor_; +}; + +// Reacts to QP being too high or too low. For best quality, when QP is high it +// is desired to decrease the resolution or frame rate of the stream and when QP +// is low it is desired to increase the resolution or frame rate of the stream. +// Whether to reconfigure the stream is ultimately up to the handler, which is +// able to respond asynchronously. +class QualityScalerQpUsageHandlerInterface { + public: + virtual ~QualityScalerQpUsageHandlerInterface(); + + virtual void OnReportQpUsageHigh() = 0; + virtual void OnReportQpUsageLow() = 0; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/quality_scaler_unittest.cc b/third_party/libwebrtc/modules/video_coding/utility/quality_scaler_unittest.cc new file mode 100644 index 0000000000..1142947d54 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/quality_scaler_unittest.cc @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/quality_scaler.h" + +#include <memory> +#include <string> + +#include "api/field_trials_view.h" +#include "api/units/time_delta.h" +#include "rtc_base/checks.h" +#include "rtc_base/event.h" +#include "rtc_base/task_queue_for_test.h" +#include "test/gtest.h" +#include "test/scoped_key_value_config.h" + +namespace webrtc { +namespace { +static const int kFramerate = 30; +static const int kLowQp = 15; +static const int kHighQp = 40; +static const int kMinFramesNeededToScale = 60; // From quality_scaler.cc. +static constexpr TimeDelta kDefaultTimeout = TimeDelta::Millis(150); +} // namespace + +class FakeQpUsageHandler : public QualityScalerQpUsageHandlerInterface { + public: + ~FakeQpUsageHandler() override = default; + + // QualityScalerQpUsageHandlerInterface implementation. + void OnReportQpUsageHigh() override { + adapt_down_events_++; + event.Set(); + } + + void OnReportQpUsageLow() override { + adapt_up_events_++; + event.Set(); + } + + rtc::Event event; + int adapt_up_events_ = 0; + int adapt_down_events_ = 0; +}; + +// Pass a lower sampling period to speed up the tests. +class QualityScalerUnderTest : public QualityScaler { + public: + explicit QualityScalerUnderTest(QualityScalerQpUsageHandlerInterface* handler, + VideoEncoder::QpThresholds thresholds, + const FieldTrialsView& field_trials) + : QualityScaler(handler, thresholds, field_trials, 5) {} +}; + +class QualityScalerTest : public ::testing::Test, + public ::testing::WithParamInterface<std::string> { + protected: + enum ScaleDirection { + kKeepScaleAboveLowQp, + kKeepScaleAtHighQp, + kScaleDown, + kScaleDownAboveHighQp, + kScaleUp + }; + + QualityScalerTest() + : scoped_field_trial_(GetParam()), + task_queue_("QualityScalerTestQueue"), + handler_(std::make_unique<FakeQpUsageHandler>()) { + task_queue_.SendTask([this] { + qs_ = std::unique_ptr<QualityScaler>(new QualityScalerUnderTest( + handler_.get(), VideoEncoder::QpThresholds(kLowQp, kHighQp), + scoped_field_trial_)); + }); + } + + ~QualityScalerTest() override { + task_queue_.SendTask([this] { qs_ = nullptr; }); + } + + void TriggerScale(ScaleDirection scale_direction) { + for (int i = 0; i < kFramerate * 5; ++i) { + switch (scale_direction) { + case kKeepScaleAboveLowQp: + qs_->ReportQp(kLowQp + 1, 0); + break; + case kScaleUp: + qs_->ReportQp(kLowQp, 0); + break; + case kScaleDown: + qs_->ReportDroppedFrameByMediaOpt(); + break; + case kKeepScaleAtHighQp: + qs_->ReportQp(kHighQp, 0); + break; + case kScaleDownAboveHighQp: + qs_->ReportQp(kHighQp + 1, 0); + break; + } + } + } + + test::ScopedKeyValueConfig scoped_field_trial_; + TaskQueueForTest task_queue_; + std::unique_ptr<QualityScaler> qs_; + std::unique_ptr<FakeQpUsageHandler> handler_; +}; + +INSTANTIATE_TEST_SUITE_P( + FieldTrials, + QualityScalerTest, + ::testing::Values( + "WebRTC-Video-QualityScaling/Enabled-1,2,3,4,5,6,7,8,0.9,0.99,1/", + "WebRTC-Video-QualityScaling/Disabled/")); + +TEST_P(QualityScalerTest, DownscalesAfterContinuousFramedrop) { + task_queue_.SendTask([this] { TriggerScale(kScaleDown); }); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeout)); + EXPECT_EQ(1, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); +} + +TEST_P(QualityScalerTest, KeepsScaleAtHighQp) { + task_queue_.SendTask([this] { TriggerScale(kKeepScaleAtHighQp); }); + EXPECT_FALSE(handler_->event.Wait(kDefaultTimeout)); + EXPECT_EQ(0, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); +} + +TEST_P(QualityScalerTest, DownscalesAboveHighQp) { + task_queue_.SendTask([this] { TriggerScale(kScaleDownAboveHighQp); }); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeout)); + EXPECT_EQ(1, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); +} + +TEST_P(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) { + task_queue_.SendTask([this] { + for (int i = 0; i < kFramerate * 5; ++i) { + qs_->ReportDroppedFrameByMediaOpt(); + qs_->ReportDroppedFrameByMediaOpt(); + qs_->ReportQp(kHighQp, 0); + } + }); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeout)); + EXPECT_EQ(1, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); +} + +TEST_P(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) { + task_queue_.SendTask([this] { + for (int i = 0; i < kFramerate * 5; ++i) { + qs_->ReportDroppedFrameByMediaOpt(); + qs_->ReportQp(kHighQp, 0); + } + }); + EXPECT_FALSE(handler_->event.Wait(kDefaultTimeout)); + EXPECT_EQ(0, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); +} + +TEST_P(QualityScalerTest, DownscalesAfterTwoThirdsIfFieldTrialEnabled) { + const bool kDownScaleExpected = + GetParam().find("Enabled") != std::string::npos; + task_queue_.SendTask([this] { + for (int i = 0; i < kFramerate * 5; ++i) { + qs_->ReportDroppedFrameByMediaOpt(); + qs_->ReportDroppedFrameByEncoder(); + qs_->ReportQp(kHighQp, 0); + } + }); + EXPECT_EQ(kDownScaleExpected, handler_->event.Wait(kDefaultTimeout)); + EXPECT_EQ(kDownScaleExpected ? 1 : 0, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); +} + +TEST_P(QualityScalerTest, KeepsScaleOnNormalQp) { + task_queue_.SendTask([this] { TriggerScale(kKeepScaleAboveLowQp); }); + EXPECT_FALSE(handler_->event.Wait(kDefaultTimeout)); + EXPECT_EQ(0, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); +} + +TEST_P(QualityScalerTest, UpscalesAfterLowQp) { + task_queue_.SendTask([this] { TriggerScale(kScaleUp); }); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeout)); + EXPECT_EQ(0, handler_->adapt_down_events_); + EXPECT_EQ(1, handler_->adapt_up_events_); +} + +TEST_P(QualityScalerTest, ScalesDownAndBackUp) { + task_queue_.SendTask([this] { TriggerScale(kScaleDown); }); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeout)); + EXPECT_EQ(1, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); + task_queue_.SendTask([this] { TriggerScale(kScaleUp); }); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeout)); + EXPECT_EQ(1, handler_->adapt_down_events_); + EXPECT_EQ(1, handler_->adapt_up_events_); +} + +TEST_P(QualityScalerTest, DoesNotScaleUntilEnoughFramesObserved) { + task_queue_.SendTask([this] { + // Not enough frames to make a decision. + for (int i = 0; i < kMinFramesNeededToScale - 1; ++i) { + qs_->ReportQp(kLowQp, 0); + } + }); + EXPECT_FALSE(handler_->event.Wait(kDefaultTimeout)); + task_queue_.SendTask([this] { + // Send 1 more. Enough frames observed, should result in an adapt + // request. + qs_->ReportQp(kLowQp, 0); + }); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeout)); + EXPECT_EQ(0, handler_->adapt_down_events_); + EXPECT_EQ(1, handler_->adapt_up_events_); + + // Samples should be cleared after an adapt request. + task_queue_.SendTask([this] { + // Not enough frames to make a decision. + qs_->ReportQp(kLowQp, 0); + }); + EXPECT_FALSE(handler_->event.Wait(kDefaultTimeout)); + EXPECT_EQ(0, handler_->adapt_down_events_); + EXPECT_EQ(1, handler_->adapt_up_events_); +} + +TEST_P(QualityScalerTest, ScalesDownAndBackUpWithMinFramesNeeded) { + task_queue_.SendTask([this] { + for (int i = 0; i < kMinFramesNeededToScale; ++i) { + qs_->ReportQp(kHighQp + 1, 0); + } + }); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeout)); + EXPECT_EQ(1, handler_->adapt_down_events_); + EXPECT_EQ(0, handler_->adapt_up_events_); + // Samples cleared. + task_queue_.SendTask([this] { + for (int i = 0; i < kMinFramesNeededToScale; ++i) { + qs_->ReportQp(kLowQp, 0); + } + }); + EXPECT_TRUE(handler_->event.Wait(kDefaultTimeout)); + EXPECT_EQ(1, handler_->adapt_down_events_); + EXPECT_EQ(1, handler_->adapt_up_events_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/simulcast_rate_allocator.cc b/third_party/libwebrtc/modules/video_coding/utility/simulcast_rate_allocator.cc new file mode 100644 index 0000000000..1496934e1c --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/simulcast_rate_allocator.cc @@ -0,0 +1,343 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/simulcast_rate_allocator.h" + +#include <stdio.h> + +#include <algorithm> +#include <cmath> +#include <cstdint> +#include <numeric> +#include <string> +#include <tuple> +#include <vector> + +#include "rtc_base/checks.h" +#include "rtc_base/experiments/rate_control_settings.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { +namespace { +// Ratio allocation between temporal streams: +// Values as required for the VP8 codec (accumulating). +static const float + kLayerRateAllocation[kMaxTemporalStreams][kMaxTemporalStreams] = { + {1.0f, 1.0f, 1.0f, 1.0f}, // 1 layer + {0.6f, 1.0f, 1.0f, 1.0f}, // 2 layers {60%, 40%} + {0.4f, 0.6f, 1.0f, 1.0f}, // 3 layers {40%, 20%, 40%} + {0.25f, 0.4f, 0.6f, 1.0f} // 4 layers {25%, 15%, 20%, 40%} +}; + +static const float kBaseHeavy3TlRateAllocation[kMaxTemporalStreams] = { + 0.6f, 0.8f, 1.0f, 1.0f // 3 layers {60%, 20%, 20%} +}; + +const uint32_t kLegacyScreenshareTl0BitrateKbps = 200; +const uint32_t kLegacyScreenshareTl1BitrateKbps = 1000; +} // namespace + +float SimulcastRateAllocator::GetTemporalRateAllocation( + int num_layers, + int temporal_id, + bool base_heavy_tl3_alloc) { + RTC_CHECK_GT(num_layers, 0); + RTC_CHECK_LE(num_layers, kMaxTemporalStreams); + RTC_CHECK_GE(temporal_id, 0); + RTC_CHECK_LT(temporal_id, num_layers); + if (num_layers == 3 && base_heavy_tl3_alloc) { + return kBaseHeavy3TlRateAllocation[temporal_id]; + } + return kLayerRateAllocation[num_layers - 1][temporal_id]; +} + +SimulcastRateAllocator::SimulcastRateAllocator(const VideoCodec& codec) + : codec_(codec), + stable_rate_settings_(StableTargetRateExperiment::ParseFromFieldTrials()), + rate_control_settings_(RateControlSettings::ParseFromFieldTrials()), + legacy_conference_mode_(false) {} + +SimulcastRateAllocator::~SimulcastRateAllocator() = default; + +VideoBitrateAllocation SimulcastRateAllocator::Allocate( + VideoBitrateAllocationParameters parameters) { + VideoBitrateAllocation allocated_bitrates; + DataRate stable_rate = parameters.total_bitrate; + if (stable_rate_settings_.IsEnabled() && + parameters.stable_bitrate > DataRate::Zero()) { + stable_rate = std::min(parameters.stable_bitrate, parameters.total_bitrate); + } + DistributeAllocationToSimulcastLayers(parameters.total_bitrate, stable_rate, + &allocated_bitrates); + DistributeAllocationToTemporalLayers(&allocated_bitrates); + return allocated_bitrates; +} + +void SimulcastRateAllocator::DistributeAllocationToSimulcastLayers( + DataRate total_bitrate, + DataRate stable_bitrate, + VideoBitrateAllocation* allocated_bitrates) { + DataRate left_in_total_allocation = total_bitrate; + DataRate left_in_stable_allocation = stable_bitrate; + + if (codec_.maxBitrate) { + DataRate max_rate = DataRate::KilobitsPerSec(codec_.maxBitrate); + left_in_total_allocation = std::min(left_in_total_allocation, max_rate); + left_in_stable_allocation = std::min(left_in_stable_allocation, max_rate); + } + + if (codec_.numberOfSimulcastStreams == 0) { + // No simulcast, just set the target as this has been capped already. + if (codec_.active) { + allocated_bitrates->SetBitrate( + 0, 0, + std::max(DataRate::KilobitsPerSec(codec_.minBitrate), + left_in_total_allocation) + .bps()); + } + return; + } + + // Sort the layers by maxFramerate, they might not always be from smallest + // to biggest + std::vector<size_t> layer_index(codec_.numberOfSimulcastStreams); + std::iota(layer_index.begin(), layer_index.end(), 0); + std::stable_sort(layer_index.begin(), layer_index.end(), + [this](size_t a, size_t b) { + return std::tie(codec_.simulcastStream[a].maxBitrate) < + std::tie(codec_.simulcastStream[b].maxBitrate); + }); + + // Find the first active layer. We don't allocate to inactive layers. + size_t active_layer = 0; + for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) { + if (codec_.simulcastStream[layer_index[active_layer]].active) { + // Found the first active layer. + break; + } + } + // All streams could be inactive, and nothing more to do. + if (active_layer == codec_.numberOfSimulcastStreams) { + return; + } + + // Always allocate enough bitrate for the minimum bitrate of the first + // active layer. Suspending below min bitrate is controlled outside the + // codec implementation and is not overridden by this. + DataRate min_rate = DataRate::KilobitsPerSec( + codec_.simulcastStream[layer_index[active_layer]].minBitrate); + left_in_total_allocation = std::max(left_in_total_allocation, min_rate); + left_in_stable_allocation = std::max(left_in_stable_allocation, min_rate); + + // Begin by allocating bitrate to simulcast streams, putting all bitrate in + // temporal layer 0. We'll then distribute this bitrate, across potential + // temporal layers, when stream allocation is done. + + bool first_allocation = false; + if (stream_enabled_.empty()) { + // First time allocating, this means we should not include hysteresis in + // case this is a reconfiguration of an existing enabled stream. + first_allocation = true; + stream_enabled_.resize(codec_.numberOfSimulcastStreams, false); + } + + size_t top_active_layer = active_layer; + // Allocate up to the target bitrate for each active simulcast layer. + for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) { + const SimulcastStream& stream = + codec_.simulcastStream[layer_index[active_layer]]; + if (!stream.active) { + stream_enabled_[layer_index[active_layer]] = false; + continue; + } + // If we can't allocate to the current layer we can't allocate to higher + // layers because they require a higher minimum bitrate. + DataRate min_bitrate = DataRate::KilobitsPerSec(stream.minBitrate); + DataRate target_bitrate = DataRate::KilobitsPerSec(stream.targetBitrate); + double hysteresis_factor = + codec_.mode == VideoCodecMode::kRealtimeVideo + ? stable_rate_settings_.GetVideoHysteresisFactor() + : stable_rate_settings_.GetScreenshareHysteresisFactor(); + if (!first_allocation && !stream_enabled_[layer_index[active_layer]]) { + min_bitrate = std::min(hysteresis_factor * min_bitrate, target_bitrate); + } + if (left_in_stable_allocation < min_bitrate) { + allocated_bitrates->set_bw_limited(true); + break; + } + + // We are allocating to this layer so it is the current active allocation. + top_active_layer = layer_index[active_layer]; + stream_enabled_[layer_index[active_layer]] = true; + DataRate layer_rate = std::min(left_in_total_allocation, target_bitrate); + allocated_bitrates->SetBitrate(layer_index[active_layer], 0, + layer_rate.bps()); + left_in_total_allocation -= layer_rate; + left_in_stable_allocation -= + std::min(left_in_stable_allocation, target_bitrate); + } + + // All layers above this one are not active. + for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) { + stream_enabled_[layer_index[active_layer]] = false; + } + + // Next, try allocate remaining bitrate, up to max bitrate, in top active + // stream. + // TODO(sprang): Allocate up to max bitrate for all layers once we have a + // better idea of possible performance implications. + if (left_in_total_allocation > DataRate::Zero()) { + const SimulcastStream& stream = codec_.simulcastStream[top_active_layer]; + DataRate initial_layer_rate = DataRate::BitsPerSec( + allocated_bitrates->GetSpatialLayerSum(top_active_layer)); + DataRate additional_allocation = std::min( + left_in_total_allocation, + DataRate::KilobitsPerSec(stream.maxBitrate) - initial_layer_rate); + allocated_bitrates->SetBitrate( + top_active_layer, 0, + (initial_layer_rate + additional_allocation).bps()); + } +} + +void SimulcastRateAllocator::DistributeAllocationToTemporalLayers( + VideoBitrateAllocation* allocated_bitrates_bps) const { + const int num_spatial_streams = + std::max(1, static_cast<int>(codec_.numberOfSimulcastStreams)); + + // Finally, distribute the bitrate for the simulcast streams across the + // available temporal layers. + for (int simulcast_id = 0; simulcast_id < num_spatial_streams; + ++simulcast_id) { + uint32_t target_bitrate_kbps = + allocated_bitrates_bps->GetBitrate(simulcast_id, 0) / 1000; + if (target_bitrate_kbps == 0) { + continue; + } + + const uint32_t expected_allocated_bitrate_kbps = target_bitrate_kbps; + RTC_DCHECK_EQ( + target_bitrate_kbps, + allocated_bitrates_bps->GetSpatialLayerSum(simulcast_id) / 1000); + const int num_temporal_streams = NumTemporalStreams(simulcast_id); + uint32_t max_bitrate_kbps; + // Legacy temporal-layered only screenshare, or simulcast screenshare + // with legacy mode for simulcast stream 0. + if (codec_.mode == VideoCodecMode::kScreensharing && + legacy_conference_mode_ && simulcast_id == 0) { + // TODO(holmer): This is a "temporary" hack for screensharing, where we + // interpret the startBitrate as the encoder target bitrate. This is + // to allow for a different max bitrate, so if the codec can't meet + // the target we still allow it to overshoot up to the max before dropping + // frames. This hack should be improved. + max_bitrate_kbps = + std::min(kLegacyScreenshareTl1BitrateKbps, target_bitrate_kbps); + target_bitrate_kbps = + std::min(kLegacyScreenshareTl0BitrateKbps, target_bitrate_kbps); + } else if (num_spatial_streams == 1) { + max_bitrate_kbps = codec_.maxBitrate; + } else { + max_bitrate_kbps = codec_.simulcastStream[simulcast_id].maxBitrate; + } + + std::vector<uint32_t> tl_allocation; + if (num_temporal_streams == 1) { + tl_allocation.push_back(target_bitrate_kbps); + } else { + if (codec_.mode == VideoCodecMode::kScreensharing && + legacy_conference_mode_ && simulcast_id == 0) { + tl_allocation = ScreenshareTemporalLayerAllocation( + target_bitrate_kbps, max_bitrate_kbps, simulcast_id); + } else { + tl_allocation = DefaultTemporalLayerAllocation( + target_bitrate_kbps, max_bitrate_kbps, simulcast_id); + } + } + RTC_DCHECK_GT(tl_allocation.size(), 0); + RTC_DCHECK_LE(tl_allocation.size(), num_temporal_streams); + + uint64_t tl_allocation_sum_kbps = 0; + for (size_t tl_index = 0; tl_index < tl_allocation.size(); ++tl_index) { + uint32_t layer_rate_kbps = tl_allocation[tl_index]; + if (layer_rate_kbps > 0) { + allocated_bitrates_bps->SetBitrate(simulcast_id, tl_index, + layer_rate_kbps * 1000); + } + tl_allocation_sum_kbps += layer_rate_kbps; + } + RTC_DCHECK_LE(tl_allocation_sum_kbps, expected_allocated_bitrate_kbps); + } +} + +std::vector<uint32_t> SimulcastRateAllocator::DefaultTemporalLayerAllocation( + int bitrate_kbps, + int max_bitrate_kbps, + int simulcast_id) const { + const size_t num_temporal_layers = NumTemporalStreams(simulcast_id); + std::vector<uint32_t> bitrates; + for (size_t i = 0; i < num_temporal_layers; ++i) { + float layer_bitrate = + bitrate_kbps * + GetTemporalRateAllocation( + num_temporal_layers, i, + rate_control_settings_.Vp8BaseHeavyTl3RateAllocation()); + bitrates.push_back(static_cast<uint32_t>(layer_bitrate + 0.5)); + } + + // Allocation table is of aggregates, transform to individual rates. + uint32_t sum = 0; + for (size_t i = 0; i < num_temporal_layers; ++i) { + uint32_t layer_bitrate = bitrates[i]; + RTC_DCHECK_LE(sum, bitrates[i]); + bitrates[i] -= sum; + sum = layer_bitrate; + + if (sum >= static_cast<uint32_t>(bitrate_kbps)) { + // Sum adds up; any subsequent layers will be 0. + bitrates.resize(i + 1); + break; + } + } + + return bitrates; +} + +std::vector<uint32_t> +SimulcastRateAllocator::ScreenshareTemporalLayerAllocation( + int bitrate_kbps, + int max_bitrate_kbps, + int simulcast_id) const { + if (simulcast_id > 0) { + return DefaultTemporalLayerAllocation(bitrate_kbps, max_bitrate_kbps, + simulcast_id); + } + std::vector<uint32_t> allocation; + allocation.push_back(bitrate_kbps); + if (max_bitrate_kbps > bitrate_kbps) + allocation.push_back(max_bitrate_kbps - bitrate_kbps); + return allocation; +} + +const VideoCodec& webrtc::SimulcastRateAllocator::GetCodec() const { + return codec_; +} + +int SimulcastRateAllocator::NumTemporalStreams(size_t simulcast_id) const { + return std::max<uint8_t>( + 1, + codec_.codecType == kVideoCodecVP8 && codec_.numberOfSimulcastStreams == 0 + ? codec_.VP8().numberOfTemporalLayers + : codec_.simulcastStream[simulcast_id].numberOfTemporalLayers); +} + +void SimulcastRateAllocator::SetLegacyConferenceMode(bool enabled) { + legacy_conference_mode_ = enabled; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/simulcast_rate_allocator.h b/third_party/libwebrtc/modules/video_coding/utility/simulcast_rate_allocator.h new file mode 100644 index 0000000000..6f93dbde74 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/simulcast_rate_allocator.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_SIMULCAST_RATE_ALLOCATOR_H_ +#define MODULES_VIDEO_CODING_UTILITY_SIMULCAST_RATE_ALLOCATOR_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <vector> + +#include "api/video/video_bitrate_allocation.h" +#include "api/video/video_bitrate_allocator.h" +#include "api/video_codecs/video_codec.h" +#include "rtc_base/experiments/rate_control_settings.h" +#include "rtc_base/experiments/stable_target_rate_experiment.h" + +namespace webrtc { + +class SimulcastRateAllocator : public VideoBitrateAllocator { + public: + explicit SimulcastRateAllocator(const VideoCodec& codec); + ~SimulcastRateAllocator() override; + + SimulcastRateAllocator(const SimulcastRateAllocator&) = delete; + SimulcastRateAllocator& operator=(const SimulcastRateAllocator&) = delete; + + VideoBitrateAllocation Allocate( + VideoBitrateAllocationParameters parameters) override; + const VideoCodec& GetCodec() const; + + static float GetTemporalRateAllocation(int num_layers, + int temporal_id, + bool base_heavy_tl3_alloc); + + void SetLegacyConferenceMode(bool mode) override; + + private: + void DistributeAllocationToSimulcastLayers( + DataRate total_bitrate, + DataRate stable_bitrate, + VideoBitrateAllocation* allocated_bitrates); + void DistributeAllocationToTemporalLayers( + VideoBitrateAllocation* allocated_bitrates) const; + std::vector<uint32_t> DefaultTemporalLayerAllocation(int bitrate_kbps, + int max_bitrate_kbps, + int simulcast_id) const; + std::vector<uint32_t> ScreenshareTemporalLayerAllocation( + int bitrate_kbps, + int max_bitrate_kbps, + int simulcast_id) const; + int NumTemporalStreams(size_t simulcast_id) const; + + const VideoCodec codec_; + const StableTargetRateExperiment stable_rate_settings_; + const RateControlSettings rate_control_settings_; + std::vector<bool> stream_enabled_; + bool legacy_conference_mode_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_UTILITY_SIMULCAST_RATE_ALLOCATOR_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc b/third_party/libwebrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc new file mode 100644 index 0000000000..24d7c58bcd --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc @@ -0,0 +1,824 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/simulcast_rate_allocator.h" + +#include <limits> +#include <memory> +#include <utility> +#include <vector> + +#include "api/video_codecs/vp8_frame_buffer_controller.h" +#include "api/video_codecs/vp8_frame_config.h" +#include "api/video_codecs/vp8_temporal_layers.h" +#include "rtc_base/checks.h" +#include "test/field_trial.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { +using ::testing::_; + +constexpr uint32_t kFramerateFps = 5; +constexpr uint32_t kMinBitrateKbps = 50; +// These correspond to kLegacyScreenshareTl(0|1)BitrateKbps in cc. +constexpr uint32_t kLegacyScreenshareTargetBitrateKbps = 200; +constexpr uint32_t kLegacyScreenshareMaxBitrateKbps = 1000; +// Bitrates for upper simulcast screenshare layer. +constexpr uint32_t kSimulcastScreenshareMinBitrateKbps = 600; +constexpr uint32_t kSimulcastScreenshareMaxBitrateKbps = 1250; +// Default video hysteresis factor: allocatable bitrate for next layer must +// exceed 20% of min setting in order to be initially turned on. +const double kDefaultHysteresis = 1.2; + +class MockTemporalLayers : public Vp8FrameBufferController { + public: + MOCK_METHOD(Vp8FrameConfig, NextFrameConfig, (size_t, uint32_t), (override)); + MOCK_METHOD(void, + OnRatesUpdated, + (size_t, const std::vector<uint32_t>&, int), + (override)); + MOCK_METHOD(Vp8EncoderConfig, UpdateConfiguration, (size_t), (override)); + MOCK_METHOD(void, + OnEncodeDone, + (size_t, uint32_t, size_t, bool, int, CodecSpecificInfo*), + (override)); +}; +} // namespace + +class SimulcastRateAllocatorTest : public ::testing::TestWithParam<bool> { + public: + SimulcastRateAllocatorTest() { + codec_.codecType = kVideoCodecVP8; + codec_.minBitrate = kMinBitrateKbps; + codec_.maxBitrate = kLegacyScreenshareMaxBitrateKbps; + codec_.active = true; + CreateAllocator(); + } + virtual ~SimulcastRateAllocatorTest() {} + + template <size_t S> + void ExpectEqual(uint32_t (&expected)[S], + const std::vector<uint32_t>& actual) { + EXPECT_EQ(S, actual.size()); + for (size_t i = 0; i < S; ++i) + EXPECT_EQ(expected[i], actual[i]) << "Mismatch at index " << i; + } + + template <size_t S> + void ExpectEqual(uint32_t (&expected)[S], + const VideoBitrateAllocation& actual) { + // EXPECT_EQ(S, actual.size()); + uint32_t sum = 0; + for (size_t i = 0; i < S; ++i) { + uint32_t layer_bitrate = actual.GetSpatialLayerSum(i); + if (layer_bitrate == 0) { + EXPECT_FALSE(actual.IsSpatialLayerUsed(i)); + } + EXPECT_EQ(expected[i] * 1000U, layer_bitrate) + << "Mismatch at index " << i; + sum += layer_bitrate; + } + EXPECT_EQ(sum, actual.get_sum_bps()); + } + + void CreateAllocator(bool legacy_conference_mode = false) { + allocator_.reset(new SimulcastRateAllocator(codec_)); + allocator_->SetLegacyConferenceMode(legacy_conference_mode); + } + + void SetupCodec3SL3TL(const std::vector<bool>& active_streams) { + const size_t num_simulcast_layers = 3; + RTC_DCHECK_GE(active_streams.size(), num_simulcast_layers); + SetupCodec2SL3TL(active_streams); + codec_.numberOfSimulcastStreams = num_simulcast_layers; + codec_.simulcastStream[2].numberOfTemporalLayers = 3; + codec_.simulcastStream[2].maxBitrate = 4000; + codec_.simulcastStream[2].targetBitrate = 3000; + codec_.simulcastStream[2].minBitrate = 2000; + codec_.simulcastStream[2].active = active_streams[2]; + } + + void SetupCodec2SL3TL(const std::vector<bool>& active_streams) { + const size_t num_simulcast_layers = 2; + RTC_DCHECK_GE(active_streams.size(), num_simulcast_layers); + SetupCodec1SL3TL(active_streams); + codec_.numberOfSimulcastStreams = num_simulcast_layers; + codec_.simulcastStream[1].numberOfTemporalLayers = 3; + codec_.simulcastStream[1].maxBitrate = 1000; + codec_.simulcastStream[1].targetBitrate = 500; + codec_.simulcastStream[1].minBitrate = 50; + codec_.simulcastStream[1].active = active_streams[1]; + } + + void SetupCodec1SL3TL(const std::vector<bool>& active_streams) { + const size_t num_simulcast_layers = 2; + RTC_DCHECK_GE(active_streams.size(), num_simulcast_layers); + SetupCodec3TL(); + codec_.numberOfSimulcastStreams = num_simulcast_layers; + codec_.simulcastStream[0].numberOfTemporalLayers = 3; + codec_.simulcastStream[0].maxBitrate = 500; + codec_.simulcastStream[0].targetBitrate = 100; + codec_.simulcastStream[0].minBitrate = 10; + codec_.simulcastStream[0].active = active_streams[0]; + } + + void SetupCodec3TL() { + codec_.maxBitrate = 0; + codec_.VP8()->numberOfTemporalLayers = 3; + } + + VideoBitrateAllocation GetAllocation(uint32_t target_bitrate) { + return allocator_->Allocate(VideoBitrateAllocationParameters( + DataRate::KilobitsPerSec(target_bitrate), kDefaultFrameRate)); + } + + VideoBitrateAllocation GetAllocation(DataRate target_rate, + DataRate stable_rate) { + return allocator_->Allocate(VideoBitrateAllocationParameters( + target_rate, stable_rate, kDefaultFrameRate)); + } + + DataRate MinRate(size_t layer_index) const { + return DataRate::KilobitsPerSec( + codec_.simulcastStream[layer_index].minBitrate); + } + + DataRate TargetRate(size_t layer_index) const { + return DataRate::KilobitsPerSec( + codec_.simulcastStream[layer_index].targetBitrate); + } + + DataRate MaxRate(size_t layer_index) const { + return DataRate::KilobitsPerSec( + codec_.simulcastStream[layer_index].maxBitrate); + } + + protected: + static const int kDefaultFrameRate = 30; + VideoCodec codec_; + std::unique_ptr<SimulcastRateAllocator> allocator_; +}; + +TEST_F(SimulcastRateAllocatorTest, NoSimulcastBelowMin) { + uint32_t expected[] = {codec_.minBitrate}; + codec_.active = true; + ExpectEqual(expected, GetAllocation(codec_.minBitrate - 1)); + ExpectEqual(expected, GetAllocation(1)); + ExpectEqual(expected, GetAllocation(0)); +} + +TEST_F(SimulcastRateAllocatorTest, NoSimulcastAboveMax) { + uint32_t expected[] = {codec_.maxBitrate}; + codec_.active = true; + ExpectEqual(expected, GetAllocation(codec_.maxBitrate + 1)); + ExpectEqual(expected, GetAllocation(std::numeric_limits<uint32_t>::max())); +} + +TEST_F(SimulcastRateAllocatorTest, NoSimulcastNoMax) { + const uint32_t kMax = VideoBitrateAllocation::kMaxBitrateBps / 1000; + codec_.active = true; + codec_.maxBitrate = 0; + CreateAllocator(); + + uint32_t expected[] = {kMax}; + ExpectEqual(expected, GetAllocation(kMax)); +} + +TEST_F(SimulcastRateAllocatorTest, NoSimulcastWithinLimits) { + codec_.active = true; + for (uint32_t bitrate = codec_.minBitrate; bitrate <= codec_.maxBitrate; + ++bitrate) { + uint32_t expected[] = {bitrate}; + ExpectEqual(expected, GetAllocation(bitrate)); + } +} + +// Tests that when we aren't using simulcast and the codec is marked inactive no +// bitrate will be allocated. +TEST_F(SimulcastRateAllocatorTest, NoSimulcastInactive) { + codec_.active = false; + uint32_t expected[] = {0}; + CreateAllocator(); + + ExpectEqual(expected, GetAllocation(kMinBitrateKbps - 10)); + ExpectEqual(expected, GetAllocation(kLegacyScreenshareTargetBitrateKbps)); + ExpectEqual(expected, GetAllocation(kLegacyScreenshareMaxBitrateKbps + 10)); +} + +TEST_F(SimulcastRateAllocatorTest, SingleSimulcastBelowMin) { + // With simulcast, use the min bitrate from the ss spec instead of the global. + codec_.numberOfSimulcastStreams = 1; + const uint32_t kMin = codec_.minBitrate - 10; + codec_.simulcastStream[0].minBitrate = kMin; + codec_.simulcastStream[0].targetBitrate = kLegacyScreenshareTargetBitrateKbps; + codec_.simulcastStream[0].active = true; + CreateAllocator(); + + uint32_t expected[] = {kMin}; + ExpectEqual(expected, GetAllocation(kMin - 1)); + ExpectEqual(expected, GetAllocation(1)); + ExpectEqual(expected, GetAllocation(0)); +} + +TEST_F(SimulcastRateAllocatorTest, SignalsBwLimited) { + // Enough to enable all layers. + const int kVeryBigBitrate = 100000; + + // With simulcast, use the min bitrate from the ss spec instead of the global. + SetupCodec3SL3TL({true, true, true}); + CreateAllocator(); + + EXPECT_TRUE( + GetAllocation(codec_.simulcastStream[0].minBitrate - 10).is_bw_limited()); + EXPECT_TRUE( + GetAllocation(codec_.simulcastStream[0].targetBitrate).is_bw_limited()); + EXPECT_TRUE(GetAllocation(codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].minBitrate) + .is_bw_limited()); + EXPECT_FALSE( + GetAllocation( + codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].targetBitrate + + static_cast<uint32_t>( + codec_.simulcastStream[2].minBitrate * kDefaultHysteresis + 0.5)) + .is_bw_limited()); + EXPECT_FALSE(GetAllocation(kVeryBigBitrate).is_bw_limited()); +} + +TEST_F(SimulcastRateAllocatorTest, SingleSimulcastAboveMax) { + codec_.numberOfSimulcastStreams = 1; + codec_.simulcastStream[0].minBitrate = kMinBitrateKbps; + const uint32_t kMax = codec_.simulcastStream[0].maxBitrate + 1000; + codec_.simulcastStream[0].maxBitrate = kMax; + codec_.simulcastStream[0].active = true; + CreateAllocator(); + + uint32_t expected[] = {kMax}; + ExpectEqual(expected, GetAllocation(kMax)); + ExpectEqual(expected, GetAllocation(kMax + 1)); + ExpectEqual(expected, GetAllocation(std::numeric_limits<uint32_t>::max())); +} + +TEST_F(SimulcastRateAllocatorTest, SingleSimulcastWithinLimits) { + codec_.numberOfSimulcastStreams = 1; + codec_.simulcastStream[0].minBitrate = kMinBitrateKbps; + codec_.simulcastStream[0].targetBitrate = kLegacyScreenshareTargetBitrateKbps; + codec_.simulcastStream[0].maxBitrate = kLegacyScreenshareMaxBitrateKbps; + codec_.simulcastStream[0].active = true; + CreateAllocator(); + + for (uint32_t bitrate = kMinBitrateKbps; + bitrate <= kLegacyScreenshareMaxBitrateKbps; ++bitrate) { + uint32_t expected[] = {bitrate}; + ExpectEqual(expected, GetAllocation(bitrate)); + } +} + +TEST_F(SimulcastRateAllocatorTest, Regular3TLTemporalRateAllocation) { + SetupCodec3SL3TL({true, true, true}); + CreateAllocator(); + + const VideoBitrateAllocation alloc = GetAllocation(kMinBitrateKbps); + // 40/20/40. + EXPECT_EQ(static_cast<uint32_t>(0.4 * kMinBitrateKbps), + alloc.GetBitrate(0, 0) / 1000); + EXPECT_EQ(static_cast<uint32_t>(0.2 * kMinBitrateKbps), + alloc.GetBitrate(0, 1) / 1000); + EXPECT_EQ(static_cast<uint32_t>(0.4 * kMinBitrateKbps), + alloc.GetBitrate(0, 2) / 1000); +} + +TEST_F(SimulcastRateAllocatorTest, BaseHeavy3TLTemporalRateAllocation) { + test::ScopedFieldTrials field_trials( + "WebRTC-UseBaseHeavyVP8TL3RateAllocation/Enabled/"); + + SetupCodec3SL3TL({true, true, true}); + CreateAllocator(); + + const VideoBitrateAllocation alloc = GetAllocation(kMinBitrateKbps); + // 60/20/20. + EXPECT_EQ(static_cast<uint32_t>(0.6 * kMinBitrateKbps), + alloc.GetBitrate(0, 0) / 1000); + EXPECT_EQ(static_cast<uint32_t>(0.2 * kMinBitrateKbps), + alloc.GetBitrate(0, 1) / 1000); + EXPECT_EQ(static_cast<uint32_t>(0.2 * kMinBitrateKbps), + alloc.GetBitrate(0, 2) / 1000); +} + +TEST_F(SimulcastRateAllocatorTest, SingleSimulcastInactive) { + codec_.numberOfSimulcastStreams = 1; + codec_.simulcastStream[0].minBitrate = kMinBitrateKbps; + codec_.simulcastStream[0].targetBitrate = kLegacyScreenshareTargetBitrateKbps; + codec_.simulcastStream[0].maxBitrate = kLegacyScreenshareMaxBitrateKbps; + codec_.simulcastStream[0].active = false; + CreateAllocator(); + + uint32_t expected[] = {0}; + ExpectEqual(expected, GetAllocation(kMinBitrateKbps - 10)); + ExpectEqual(expected, GetAllocation(kLegacyScreenshareTargetBitrateKbps)); + ExpectEqual(expected, GetAllocation(kLegacyScreenshareMaxBitrateKbps + 10)); +} + +TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { + SetupCodec3SL3TL({true, true, true}); + CreateAllocator(); + + { + // Single stream, min bitrate. + const uint32_t bitrate = codec_.simulcastStream[0].minBitrate; + uint32_t expected[] = {bitrate, 0, 0}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + { + // Single stream at target bitrate. + const uint32_t bitrate = codec_.simulcastStream[0].targetBitrate; + uint32_t expected[] = {bitrate, 0, 0}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + uint32_t kMinInitialRateTwoLayers = + codec_.simulcastStream[0].targetBitrate + + static_cast<uint32_t>(codec_.simulcastStream[1].minBitrate * + kDefaultHysteresis); + { + // Bitrate above target for first stream, but below min for the next one. + const uint32_t bitrate = kMinInitialRateTwoLayers - 1; + uint32_t expected[] = {bitrate, 0, 0}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + { + // Just enough for two streams. + const uint32_t bitrate = kMinInitialRateTwoLayers; + uint32_t expected[] = { + codec_.simulcastStream[0].targetBitrate, + kMinInitialRateTwoLayers - codec_.simulcastStream[0].targetBitrate, 0}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + { + // Second stream maxed out, but not enough for third. + const uint32_t bitrate = codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].maxBitrate; + uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, + codec_.simulcastStream[1].maxBitrate, 0}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + uint32_t kMinInitialRateThreeLayers = + codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].targetBitrate + + static_cast<uint32_t>(codec_.simulcastStream[2].minBitrate * + kDefaultHysteresis); + { + // First two streams maxed out, but not enough for third. Nowhere to put + // remaining bits. + const uint32_t bitrate = kMinInitialRateThreeLayers - 1; + uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, + codec_.simulcastStream[1].maxBitrate, 0}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + { + // Just enough for all three streams. + const uint32_t bitrate = kMinInitialRateThreeLayers; + uint32_t expected[] = { + codec_.simulcastStream[0].targetBitrate, + codec_.simulcastStream[1].targetBitrate, + static_cast<uint32_t>(codec_.simulcastStream[2].minBitrate * + kDefaultHysteresis)}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + { + // Third maxed out. + const uint32_t bitrate = codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].targetBitrate + + codec_.simulcastStream[2].maxBitrate; + uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, + codec_.simulcastStream[1].targetBitrate, + codec_.simulcastStream[2].maxBitrate}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + { + // Enough to max out all streams which will allocate the target amount to + // the lower streams. + const uint32_t bitrate = codec_.simulcastStream[0].maxBitrate + + codec_.simulcastStream[1].maxBitrate + + codec_.simulcastStream[2].maxBitrate; + uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, + codec_.simulcastStream[1].targetBitrate, + codec_.simulcastStream[2].maxBitrate}; + ExpectEqual(expected, GetAllocation(bitrate)); + } +} + +// If three simulcast streams that are all inactive, none of them should be +// allocated bitrate. +TEST_F(SimulcastRateAllocatorTest, ThreeStreamsInactive) { + SetupCodec3SL3TL({false, false, false}); + CreateAllocator(); + + // Just enough to allocate the min. + const uint32_t min_bitrate = codec_.simulcastStream[0].minBitrate + + codec_.simulcastStream[1].minBitrate + + codec_.simulcastStream[2].minBitrate; + // Enough bitrate to allocate target to all streams. + const uint32_t target_bitrate = codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].targetBitrate + + codec_.simulcastStream[2].targetBitrate; + // Enough bitrate to allocate max to all streams. + const uint32_t max_bitrate = codec_.simulcastStream[0].maxBitrate + + codec_.simulcastStream[1].maxBitrate + + codec_.simulcastStream[2].maxBitrate; + uint32_t expected[] = {0, 0, 0}; + ExpectEqual(expected, GetAllocation(0)); + ExpectEqual(expected, GetAllocation(min_bitrate)); + ExpectEqual(expected, GetAllocation(target_bitrate)); + ExpectEqual(expected, GetAllocation(max_bitrate)); +} + +// If there are two simulcast streams, we expect the high active stream to be +// allocated as if it is a single active stream. +TEST_F(SimulcastRateAllocatorTest, TwoStreamsLowInactive) { + SetupCodec2SL3TL({false, true}); + CreateAllocator(); + + const uint32_t kActiveStreamMinBitrate = codec_.simulcastStream[1].minBitrate; + const uint32_t kActiveStreamTargetBitrate = + codec_.simulcastStream[1].targetBitrate; + const uint32_t kActiveStreamMaxBitrate = codec_.simulcastStream[1].maxBitrate; + { + // Expect that the stream is always allocated its min bitrate. + uint32_t expected[] = {0, kActiveStreamMinBitrate}; + ExpectEqual(expected, GetAllocation(0)); + ExpectEqual(expected, GetAllocation(kActiveStreamMinBitrate - 10)); + ExpectEqual(expected, GetAllocation(kActiveStreamMinBitrate)); + } + + { + // The stream should be allocated its target bitrate. + uint32_t expected[] = {0, kActiveStreamTargetBitrate}; + ExpectEqual(expected, GetAllocation(kActiveStreamTargetBitrate)); + } + + { + // The stream should be allocated its max if the target input is sufficient. + uint32_t expected[] = {0, kActiveStreamMaxBitrate}; + ExpectEqual(expected, GetAllocation(kActiveStreamMaxBitrate)); + ExpectEqual(expected, GetAllocation(std::numeric_limits<uint32_t>::max())); + } +} + +// If there are two simulcast streams, we expect the low active stream to be +// allocated as if it is a single active stream. +TEST_F(SimulcastRateAllocatorTest, TwoStreamsHighInactive) { + SetupCodec2SL3TL({true, false}); + CreateAllocator(); + + const uint32_t kActiveStreamMinBitrate = codec_.simulcastStream[0].minBitrate; + const uint32_t kActiveStreamTargetBitrate = + codec_.simulcastStream[0].targetBitrate; + const uint32_t kActiveStreamMaxBitrate = codec_.simulcastStream[0].maxBitrate; + { + // Expect that the stream is always allocated its min bitrate. + uint32_t expected[] = {kActiveStreamMinBitrate, 0}; + ExpectEqual(expected, GetAllocation(0)); + ExpectEqual(expected, GetAllocation(kActiveStreamMinBitrate - 10)); + ExpectEqual(expected, GetAllocation(kActiveStreamMinBitrate)); + } + + { + // The stream should be allocated its target bitrate. + uint32_t expected[] = {kActiveStreamTargetBitrate, 0}; + ExpectEqual(expected, GetAllocation(kActiveStreamTargetBitrate)); + } + + { + // The stream should be allocated its max if the target input is sufficent. + uint32_t expected[] = {kActiveStreamMaxBitrate, 0}; + ExpectEqual(expected, GetAllocation(kActiveStreamMaxBitrate)); + ExpectEqual(expected, GetAllocation(std::numeric_limits<uint32_t>::max())); + } +} + +// If there are three simulcast streams and the middle stream is inactive, the +// other two streams should be allocated bitrate the same as if they are two +// active simulcast streams. +TEST_F(SimulcastRateAllocatorTest, ThreeStreamsMiddleInactive) { + SetupCodec3SL3TL({true, false, true}); + CreateAllocator(); + + { + const uint32_t kLowStreamMinBitrate = codec_.simulcastStream[0].minBitrate; + // The lowest stream should always be allocated its minimum bitrate. + uint32_t expected[] = {kLowStreamMinBitrate, 0, 0}; + ExpectEqual(expected, GetAllocation(0)); + ExpectEqual(expected, GetAllocation(kLowStreamMinBitrate - 10)); + ExpectEqual(expected, GetAllocation(kLowStreamMinBitrate)); + } + + { + // The lowest stream gets its target bitrate. + uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, 0, 0}; + ExpectEqual(expected, + GetAllocation(codec_.simulcastStream[0].targetBitrate)); + } + + { + // The lowest stream gets its max bitrate, but not enough for the high + // stream. + const uint32_t bitrate = codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[2].minBitrate - 1; + uint32_t expected[] = {codec_.simulcastStream[0].maxBitrate, 0, 0}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + { + // Both active streams get allocated target bitrate. + const uint32_t bitrate = codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[2].targetBitrate; + uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, 0, + codec_.simulcastStream[2].targetBitrate}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + { + // Lowest stream gets its target bitrate, high stream gets its max bitrate. + uint32_t bitrate = codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[2].maxBitrate; + uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, 0, + codec_.simulcastStream[2].maxBitrate}; + ExpectEqual(expected, GetAllocation(bitrate)); + ExpectEqual(expected, GetAllocation(bitrate + 10)); + ExpectEqual(expected, GetAllocation(std::numeric_limits<uint32_t>::max())); + } +} + +TEST_F(SimulcastRateAllocatorTest, NonConferenceModeScreenshare) { + codec_.mode = VideoCodecMode::kScreensharing; + SetupCodec3SL3TL({true, true, true}); + CreateAllocator(); + + // Make sure we have enough bitrate for all 3 simulcast layers + const uint32_t bitrate = codec_.simulcastStream[0].maxBitrate + + codec_.simulcastStream[1].maxBitrate + + codec_.simulcastStream[2].maxBitrate; + const VideoBitrateAllocation alloc = GetAllocation(bitrate); + + EXPECT_EQ(alloc.GetTemporalLayerAllocation(0).size(), 3u); + EXPECT_EQ(alloc.GetTemporalLayerAllocation(1).size(), 3u); + EXPECT_EQ(alloc.GetTemporalLayerAllocation(2).size(), 3u); +} + +TEST_F(SimulcastRateAllocatorTest, StableRate) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-StableTargetRate/" + "enabled:true," + "video_hysteresis_factor:1.1/"); + + SetupCodec3SL3TL({true, true, true}); + CreateAllocator(); + + // Let the volatile rate always be be enough for all streams, in this test we + // are only interested in how the stable rate affects enablement. + const DataRate volatile_rate = + (TargetRate(0) + TargetRate(1) + MinRate(2)) * 1.1; + + { + // On the first call to a new SimulcastRateAllocator instance, hysteresis + // is disabled, but stable rate still caps layers. + uint32_t expected[] = {TargetRate(0).kbps<uint32_t>(), + MaxRate(1).kbps<uint32_t>()}; + ExpectEqual(expected, + GetAllocation(volatile_rate, TargetRate(0) + MinRate(1))); + } + + { + // Let stable rate go to a bitrate below what is needed for two streams. + uint32_t expected[] = {MaxRate(0).kbps<uint32_t>(), 0}; + ExpectEqual(expected, + GetAllocation(volatile_rate, TargetRate(0) + MinRate(1) - + DataRate::BitsPerSec(1))); + } + + { + // Don't enable stream as we need to get up above hysteresis threshold. + uint32_t expected[] = {MaxRate(0).kbps<uint32_t>(), 0}; + ExpectEqual(expected, + GetAllocation(volatile_rate, TargetRate(0) + MinRate(1))); + } + + { + // Above threshold with hysteresis, enable second stream. + uint32_t expected[] = {TargetRate(0).kbps<uint32_t>(), + MaxRate(1).kbps<uint32_t>()}; + ExpectEqual(expected, GetAllocation(volatile_rate, + (TargetRate(0) + MinRate(1)) * 1.1)); + } + + { + // Enough to enable all thee layers. + uint32_t expected[] = { + TargetRate(0).kbps<uint32_t>(), TargetRate(1).kbps<uint32_t>(), + (volatile_rate - TargetRate(0) - TargetRate(1)).kbps<uint32_t>()}; + ExpectEqual(expected, GetAllocation(volatile_rate, volatile_rate)); + } + + { + // Drop hysteresis, all three still on. + uint32_t expected[] = { + TargetRate(0).kbps<uint32_t>(), TargetRate(1).kbps<uint32_t>(), + (volatile_rate - TargetRate(0) - TargetRate(1)).kbps<uint32_t>()}; + ExpectEqual(expected, + GetAllocation(volatile_rate, + TargetRate(0) + TargetRate(1) + MinRate(2))); + } +} + +class ScreenshareRateAllocationTest : public SimulcastRateAllocatorTest { + public: + void SetupConferenceScreenshare(bool use_simulcast, bool active = true) { + codec_.mode = VideoCodecMode::kScreensharing; + codec_.minBitrate = kMinBitrateKbps; + codec_.maxBitrate = + kLegacyScreenshareMaxBitrateKbps + kSimulcastScreenshareMaxBitrateKbps; + if (use_simulcast) { + codec_.numberOfSimulcastStreams = 2; + codec_.simulcastStream[0].minBitrate = kMinBitrateKbps; + codec_.simulcastStream[0].targetBitrate = + kLegacyScreenshareTargetBitrateKbps; + codec_.simulcastStream[0].maxBitrate = kLegacyScreenshareMaxBitrateKbps; + codec_.simulcastStream[0].numberOfTemporalLayers = 2; + codec_.simulcastStream[0].active = active; + + codec_.simulcastStream[1].minBitrate = + kSimulcastScreenshareMinBitrateKbps; + codec_.simulcastStream[1].targetBitrate = + kSimulcastScreenshareMaxBitrateKbps; + codec_.simulcastStream[1].maxBitrate = + kSimulcastScreenshareMaxBitrateKbps; + codec_.simulcastStream[1].numberOfTemporalLayers = 2; + codec_.simulcastStream[1].active = active; + } else { + codec_.numberOfSimulcastStreams = 0; + codec_.VP8()->numberOfTemporalLayers = 2; + codec_.active = active; + } + } +}; + +INSTANTIATE_TEST_SUITE_P(ScreenshareTest, + ScreenshareRateAllocationTest, + ::testing::Bool()); + +TEST_P(ScreenshareRateAllocationTest, ConferenceBitrateBelowTl0) { + SetupConferenceScreenshare(GetParam()); + CreateAllocator(true); + + VideoBitrateAllocation allocation = + allocator_->Allocate(VideoBitrateAllocationParameters( + kLegacyScreenshareTargetBitrateKbps * 1000, kFramerateFps)); + + // All allocation should go in TL0. + EXPECT_EQ(kLegacyScreenshareTargetBitrateKbps, allocation.get_sum_kbps()); + EXPECT_EQ(kLegacyScreenshareTargetBitrateKbps, + allocation.GetBitrate(0, 0) / 1000); + EXPECT_EQ(allocation.is_bw_limited(), GetParam()); +} + +TEST_P(ScreenshareRateAllocationTest, ConferenceBitrateAboveTl0) { + SetupConferenceScreenshare(GetParam()); + CreateAllocator(true); + + uint32_t target_bitrate_kbps = + (kLegacyScreenshareTargetBitrateKbps + kLegacyScreenshareMaxBitrateKbps) / + 2; + VideoBitrateAllocation allocation = + allocator_->Allocate(VideoBitrateAllocationParameters( + target_bitrate_kbps * 1000, kFramerateFps)); + + // Fill TL0, then put the rest in TL1. + EXPECT_EQ(target_bitrate_kbps, allocation.get_sum_kbps()); + EXPECT_EQ(kLegacyScreenshareTargetBitrateKbps, + allocation.GetBitrate(0, 0) / 1000); + EXPECT_EQ(target_bitrate_kbps - kLegacyScreenshareTargetBitrateKbps, + allocation.GetBitrate(0, 1) / 1000); + EXPECT_EQ(allocation.is_bw_limited(), GetParam()); +} + +TEST_F(ScreenshareRateAllocationTest, ConferenceBitrateAboveTl1) { + // This test is only for the non-simulcast case. + SetupConferenceScreenshare(false); + CreateAllocator(true); + + VideoBitrateAllocation allocation = + allocator_->Allocate(VideoBitrateAllocationParameters( + kLegacyScreenshareMaxBitrateKbps * 2000, kFramerateFps)); + + // Fill both TL0 and TL1, but no more. + EXPECT_EQ(kLegacyScreenshareMaxBitrateKbps, allocation.get_sum_kbps()); + EXPECT_EQ(kLegacyScreenshareTargetBitrateKbps, + allocation.GetBitrate(0, 0) / 1000); + EXPECT_EQ( + kLegacyScreenshareMaxBitrateKbps - kLegacyScreenshareTargetBitrateKbps, + allocation.GetBitrate(0, 1) / 1000); + EXPECT_FALSE(allocation.is_bw_limited()); +} + +// This tests when the screenshare is inactive it should be allocated 0 bitrate +// for all layers. +TEST_P(ScreenshareRateAllocationTest, InactiveScreenshare) { + SetupConferenceScreenshare(GetParam(), false); + CreateAllocator(); + + // Enough bitrate for TL0 and TL1. + uint32_t target_bitrate_kbps = + (kLegacyScreenshareTargetBitrateKbps + kLegacyScreenshareMaxBitrateKbps) / + 2; + VideoBitrateAllocation allocation = + allocator_->Allocate(VideoBitrateAllocationParameters( + target_bitrate_kbps * 1000, kFramerateFps)); + + EXPECT_EQ(0U, allocation.get_sum_kbps()); +} + +TEST_F(ScreenshareRateAllocationTest, Hysteresis) { + // This test is only for the simulcast case. + SetupConferenceScreenshare(true); + CreateAllocator(); + + // The bitrate at which we would normally enable the upper simulcast stream. + const uint32_t default_enable_rate_bps = + codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].minBitrate; + const uint32_t enable_rate_with_hysteresis_bps = + (default_enable_rate_bps * 135) / 100; + + { + // On the first call to a new SimulcastRateAllocator instance, hysteresis + // is disabled. + const uint32_t bitrate = default_enable_rate_bps; + uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, + codec_.simulcastStream[1].minBitrate}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + { + // Go down to a bitrate below what is needed for two streams. + const uint32_t bitrate = default_enable_rate_bps - 1; + uint32_t expected[] = {bitrate, 0}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + { + // Don't enable stream as we need to get up above hysteresis threshold. + const uint32_t bitrate = default_enable_rate_bps; + uint32_t expected[] = {bitrate, 0}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + { + // Above threshold, enable second stream. + const uint32_t bitrate = enable_rate_with_hysteresis_bps; + uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, + enable_rate_with_hysteresis_bps - + codec_.simulcastStream[0].targetBitrate}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + { + // Go down again, still keep the second stream alive. + const uint32_t bitrate = default_enable_rate_bps; + uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, + codec_.simulcastStream[1].minBitrate}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + { + // Go down below default enable, second stream is shut down again. + const uint32_t bitrate = default_enable_rate_bps - 1; + uint32_t expected[] = {bitrate, 0}; + ExpectEqual(expected, GetAllocation(bitrate)); + } + + { + // Go up, hysteresis is blocking us again. + const uint32_t bitrate = default_enable_rate_bps; + uint32_t expected[] = {bitrate, 0}; + ExpectEqual(expected, GetAllocation(bitrate)); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/simulcast_test_fixture_impl.cc b/third_party/libwebrtc/modules/video_coding/utility/simulcast_test_fixture_impl.cc new file mode 100644 index 0000000000..c6e51e8068 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/simulcast_test_fixture_impl.cc @@ -0,0 +1,967 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/simulcast_test_fixture_impl.h" + +#include <algorithm> +#include <map> +#include <memory> +#include <vector> + +#include "api/video/encoded_image.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_coding_defines.h" +#include "rtc_base/checks.h" +#include "test/gtest.h" + +using ::testing::_; +using ::testing::AllOf; +using ::testing::Field; +using ::testing::Return; + +namespace webrtc { +namespace test { + +namespace { + +const int kDefaultWidth = 1280; +const int kDefaultHeight = 720; +const int kNumberOfSimulcastStreams = 3; +const int kColorY = 66; +const int kColorU = 22; +const int kColorV = 33; +const int kMaxBitrates[kNumberOfSimulcastStreams] = {150, 600, 1200}; +const int kMinBitrates[kNumberOfSimulcastStreams] = {50, 150, 600}; +const int kTargetBitrates[kNumberOfSimulcastStreams] = {100, 450, 1000}; +const float kMaxFramerates[kNumberOfSimulcastStreams] = {30, 30, 30}; +const int kScaleResolutionDownBy[kNumberOfSimulcastStreams] = {4, 2, 1}; +const int kDefaultTemporalLayerProfile[3] = {3, 3, 3}; +const int kNoTemporalLayerProfile[3] = {0, 0, 0}; + +const VideoEncoder::Capabilities kCapabilities(false); +const VideoEncoder::Settings kSettings(kCapabilities, 1, 1200); + +template <typename T> +void SetExpectedValues3(T value0, T value1, T value2, T* expected_values) { + expected_values[0] = value0; + expected_values[1] = value1; + expected_values[2] = value2; +} + +enum PlaneType { + kYPlane = 0, + kUPlane = 1, + kVPlane = 2, + kNumOfPlanes = 3, +}; + +} // namespace + +class SimulcastTestFixtureImpl::TestEncodedImageCallback + : public EncodedImageCallback { + public: + TestEncodedImageCallback() { + memset(temporal_layer_, -1, sizeof(temporal_layer_)); + memset(layer_sync_, false, sizeof(layer_sync_)); + } + + Result OnEncodedImage(const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) override { + bool is_vp8 = (codec_specific_info->codecType == kVideoCodecVP8); + bool is_h264 = (codec_specific_info->codecType == kVideoCodecH264); + // Only store the base layer. + if (encoded_image.SimulcastIndex().value_or(0) == 0) { + if (encoded_image._frameType == VideoFrameType::kVideoFrameKey) { + encoded_key_frame_.SetEncodedData(EncodedImageBuffer::Create( + encoded_image.data(), encoded_image.size())); + encoded_key_frame_._frameType = VideoFrameType::kVideoFrameKey; + } else { + encoded_frame_.SetEncodedData(EncodedImageBuffer::Create( + encoded_image.data(), encoded_image.size())); + } + } + if (is_vp8) { + layer_sync_[encoded_image.SimulcastIndex().value_or(0)] = + codec_specific_info->codecSpecific.VP8.layerSync; + temporal_layer_[encoded_image.SimulcastIndex().value_or(0)] = + codec_specific_info->codecSpecific.VP8.temporalIdx; + } else if (is_h264) { + layer_sync_[encoded_image.SimulcastIndex().value_or(0)] = + codec_specific_info->codecSpecific.H264.base_layer_sync; + temporal_layer_[encoded_image.SimulcastIndex().value_or(0)] = + codec_specific_info->codecSpecific.H264.temporal_idx; + } + return Result(Result::OK, encoded_image.RtpTimestamp()); + } + // This method only makes sense for VP8. + void GetLastEncodedFrameInfo(int* temporal_layer, + bool* layer_sync, + int stream) { + *temporal_layer = temporal_layer_[stream]; + *layer_sync = layer_sync_[stream]; + } + void GetLastEncodedKeyFrame(EncodedImage* encoded_key_frame) { + *encoded_key_frame = encoded_key_frame_; + } + void GetLastEncodedFrame(EncodedImage* encoded_frame) { + *encoded_frame = encoded_frame_; + } + + private: + EncodedImage encoded_key_frame_; + EncodedImage encoded_frame_; + int temporal_layer_[kNumberOfSimulcastStreams]; + bool layer_sync_[kNumberOfSimulcastStreams]; +}; + +class SimulcastTestFixtureImpl::TestDecodedImageCallback + : public DecodedImageCallback { + public: + TestDecodedImageCallback() : decoded_frames_(0) {} + int32_t Decoded(VideoFrame& decoded_image) override { + rtc::scoped_refptr<I420BufferInterface> i420_buffer = + decoded_image.video_frame_buffer()->ToI420(); + for (int i = 0; i < decoded_image.width(); ++i) { + EXPECT_NEAR(kColorY, i420_buffer->DataY()[i], 1); + } + + // TODO(mikhal): Verify the difference between U,V and the original. + for (int i = 0; i < i420_buffer->ChromaWidth(); ++i) { + EXPECT_NEAR(kColorU, i420_buffer->DataU()[i], 4); + EXPECT_NEAR(kColorV, i420_buffer->DataV()[i], 4); + } + decoded_frames_++; + return 0; + } + int32_t Decoded(VideoFrame& decoded_image, int64_t decode_time_ms) override { + RTC_DCHECK_NOTREACHED(); + return -1; + } + void Decoded(VideoFrame& decoded_image, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp) override { + Decoded(decoded_image); + } + int DecodedFrames() { return decoded_frames_; } + + private: + int decoded_frames_; +}; + +namespace { + +void SetPlane(uint8_t* data, uint8_t value, int width, int height, int stride) { + for (int i = 0; i < height; i++, data += stride) { + // Setting allocated area to zero - setting only image size to + // requested values - will make it easier to distinguish between image + // size and frame size (accounting for stride). + memset(data, value, width); + memset(data + width, 0, stride - width); + } +} + +// Fills in an I420Buffer from `plane_colors`. +void CreateImage(const rtc::scoped_refptr<I420Buffer>& buffer, + int plane_colors[kNumOfPlanes]) { + SetPlane(buffer->MutableDataY(), plane_colors[0], buffer->width(), + buffer->height(), buffer->StrideY()); + + SetPlane(buffer->MutableDataU(), plane_colors[1], buffer->ChromaWidth(), + buffer->ChromaHeight(), buffer->StrideU()); + + SetPlane(buffer->MutableDataV(), plane_colors[2], buffer->ChromaWidth(), + buffer->ChromaHeight(), buffer->StrideV()); +} + +void ConfigureStream(int width, + int height, + int max_bitrate, + int min_bitrate, + int target_bitrate, + float max_framerate, + SimulcastStream* stream, + int num_temporal_layers) { + RTC_DCHECK(stream); + stream->width = width; + stream->height = height; + stream->maxBitrate = max_bitrate; + stream->minBitrate = min_bitrate; + stream->targetBitrate = target_bitrate; + stream->maxFramerate = max_framerate; + if (num_temporal_layers >= 0) { + stream->numberOfTemporalLayers = num_temporal_layers; + } + stream->qpMax = 45; + stream->active = true; +} + +} // namespace + +void SimulcastTestFixtureImpl::DefaultSettings( + VideoCodec* settings, + const int* temporal_layer_profile, + VideoCodecType codec_type, + bool reverse_layer_order) { + RTC_CHECK(settings); + *settings = {}; + settings->codecType = codec_type; + settings->startBitrate = 300; + settings->minBitrate = 30; + settings->maxBitrate = 0; + settings->maxFramerate = 30; + settings->width = kDefaultWidth; + settings->height = kDefaultHeight; + settings->numberOfSimulcastStreams = kNumberOfSimulcastStreams; + settings->active = true; + ASSERT_EQ(3, kNumberOfSimulcastStreams); + int layer_order[3] = {0, 1, 2}; + if (reverse_layer_order) { + layer_order[0] = 2; + layer_order[2] = 0; + } + settings->timing_frame_thresholds = {kDefaultTimingFramesDelayMs, + kDefaultOutlierFrameSizePercent}; + ConfigureStream(kDefaultWidth / 4, kDefaultHeight / 4, kMaxBitrates[0], + kMinBitrates[0], kTargetBitrates[0], kMaxFramerates[0], + &settings->simulcastStream[layer_order[0]], + temporal_layer_profile[0]); + ConfigureStream(kDefaultWidth / 2, kDefaultHeight / 2, kMaxBitrates[1], + kMinBitrates[1], kTargetBitrates[1], kMaxFramerates[1], + &settings->simulcastStream[layer_order[1]], + temporal_layer_profile[1]); + ConfigureStream(kDefaultWidth, kDefaultHeight, kMaxBitrates[2], + kMinBitrates[2], kTargetBitrates[2], kMaxFramerates[2], + &settings->simulcastStream[layer_order[2]], + temporal_layer_profile[2]); + settings->SetFrameDropEnabled(true); + if (codec_type == kVideoCodecVP8) { + settings->VP8()->denoisingOn = true; + settings->VP8()->automaticResizeOn = false; + settings->VP8()->keyFrameInterval = 3000; + } else { + settings->H264()->keyFrameInterval = 3000; + } +} + +SimulcastTestFixtureImpl::SimulcastTestFixtureImpl( + std::unique_ptr<VideoEncoderFactory> encoder_factory, + std::unique_ptr<VideoDecoderFactory> decoder_factory, + SdpVideoFormat video_format) + : codec_type_(PayloadStringToCodecType(video_format.name)) { + encoder_ = encoder_factory->CreateVideoEncoder(video_format); + decoder_ = decoder_factory->CreateVideoDecoder(video_format); + SetUpCodec((codec_type_ == kVideoCodecVP8 || codec_type_ == kVideoCodecH264) + ? kDefaultTemporalLayerProfile + : kNoTemporalLayerProfile); +} + +SimulcastTestFixtureImpl::~SimulcastTestFixtureImpl() { + encoder_->Release(); + decoder_->Release(); +} + +void SimulcastTestFixtureImpl::SetUpCodec(const int* temporal_layer_profile) { + encoder_->RegisterEncodeCompleteCallback(&encoder_callback_); + decoder_->RegisterDecodeCompleteCallback(&decoder_callback_); + DefaultSettings(&settings_, temporal_layer_profile, codec_type_); + SetUpRateAllocator(); + EXPECT_EQ(0, encoder_->InitEncode(&settings_, kSettings)); + VideoDecoder::Settings decoder_settings; + decoder_settings.set_max_render_resolution({kDefaultWidth, kDefaultHeight}); + decoder_settings.set_codec_type(codec_type_); + EXPECT_TRUE(decoder_->Configure(decoder_settings)); + input_buffer_ = I420Buffer::Create(kDefaultWidth, kDefaultHeight); + input_buffer_->InitializeData(); + input_frame_ = std::make_unique<webrtc::VideoFrame>( + webrtc::VideoFrame::Builder() + .set_video_frame_buffer(input_buffer_) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build()); +} + +void SimulcastTestFixtureImpl::SetUpRateAllocator() { + rate_allocator_.reset(new SimulcastRateAllocator(settings_)); +} + +void SimulcastTestFixtureImpl::SetRates(uint32_t bitrate_kbps, uint32_t fps) { + encoder_->SetRates(VideoEncoder::RateControlParameters( + rate_allocator_->Allocate( + VideoBitrateAllocationParameters(bitrate_kbps * 1000, fps)), + static_cast<double>(fps))); +} + +void SimulcastTestFixtureImpl::RunActiveStreamsTest( + const std::vector<bool> active_streams) { + std::vector<VideoFrameType> frame_types(kNumberOfSimulcastStreams, + VideoFrameType::kVideoFrameDelta); + UpdateActiveStreams(active_streams); + // Set sufficient bitrate for all streams so we can test active without + // bitrate being an issue. + SetRates(kMaxBitrates[0] + kMaxBitrates[1] + kMaxBitrates[2], 30); + + ExpectStreams(VideoFrameType::kVideoFrameKey, active_streams); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + ExpectStreams(VideoFrameType::kVideoFrameDelta, active_streams); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); +} + +void SimulcastTestFixtureImpl::UpdateActiveStreams( + const std::vector<bool> active_streams) { + ASSERT_EQ(static_cast<int>(active_streams.size()), kNumberOfSimulcastStreams); + for (size_t i = 0; i < active_streams.size(); ++i) { + settings_.simulcastStream[i].active = active_streams[i]; + } + // Re initialize the allocator and encoder with the new settings. + // TODO(bugs.webrtc.org/8807): Currently, we do a full "hard" + // reconfiguration of the allocator and encoder. When the video bitrate + // allocator has support for updating active streams without a + // reinitialization, we can just call that here instead. + SetUpRateAllocator(); + EXPECT_EQ(0, encoder_->InitEncode(&settings_, kSettings)); +} + +void SimulcastTestFixtureImpl::ExpectStream(VideoFrameType frame_type, + int scaleResolutionDownBy) { + EXPECT_CALL( + encoder_callback_, + OnEncodedImage(AllOf(Field(&EncodedImage::_frameType, frame_type), + Field(&EncodedImage::_encodedWidth, + kDefaultWidth / scaleResolutionDownBy), + Field(&EncodedImage::_encodedHeight, + kDefaultHeight / scaleResolutionDownBy)), + _)) + .Times(1) + .WillRepeatedly(Return( + EncodedImageCallback::Result(EncodedImageCallback::Result::OK, 0))); +} + +void SimulcastTestFixtureImpl::ExpectStreams( + VideoFrameType frame_type, + const std::vector<bool> expected_streams_active) { + ASSERT_EQ(static_cast<int>(expected_streams_active.size()), + kNumberOfSimulcastStreams); + for (size_t i = 0; i < kNumberOfSimulcastStreams; i++) { + if (expected_streams_active[i]) { + ExpectStream(frame_type, kScaleResolutionDownBy[i]); + } + } +} + +void SimulcastTestFixtureImpl::ExpectStreams(VideoFrameType frame_type, + int expected_video_streams) { + ASSERT_GE(expected_video_streams, 0); + ASSERT_LE(expected_video_streams, kNumberOfSimulcastStreams); + std::vector<bool> expected_streams_active(kNumberOfSimulcastStreams, false); + for (int i = 0; i < expected_video_streams; ++i) { + expected_streams_active[i] = true; + } + ExpectStreams(frame_type, expected_streams_active); +} + +void SimulcastTestFixtureImpl::VerifyTemporalIdxAndSyncForAllSpatialLayers( + TestEncodedImageCallback* encoder_callback, + const int* expected_temporal_idx, + const bool* expected_layer_sync, + int num_spatial_layers) { + int temporal_layer = -1; + bool layer_sync = false; + for (int i = 0; i < num_spatial_layers; i++) { + encoder_callback->GetLastEncodedFrameInfo(&temporal_layer, &layer_sync, i); + EXPECT_EQ(expected_temporal_idx[i], temporal_layer); + EXPECT_EQ(expected_layer_sync[i], layer_sync); + } +} + +// For some codecs (VP8) expect all active streams to generate a key frame even +// though a key frame was only requested for some of them. +void SimulcastTestFixtureImpl::TestKeyFrameRequestsOnAllStreams() { + SetRates(kMaxBitrates[2], 30); // To get all three streams. + std::vector<VideoFrameType> frame_types(kNumberOfSimulcastStreams, + VideoFrameType::kVideoFrameDelta); + ExpectStreams(VideoFrameType::kVideoFrameKey, kNumberOfSimulcastStreams); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + ExpectStreams(VideoFrameType::kVideoFrameDelta, kNumberOfSimulcastStreams); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + frame_types[0] = VideoFrameType::kVideoFrameKey; + ExpectStreams(VideoFrameType::kVideoFrameKey, kNumberOfSimulcastStreams); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + std::fill(frame_types.begin(), frame_types.end(), + VideoFrameType::kVideoFrameDelta); + frame_types[1] = VideoFrameType::kVideoFrameKey; + ExpectStreams(VideoFrameType::kVideoFrameKey, kNumberOfSimulcastStreams); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + std::fill(frame_types.begin(), frame_types.end(), + VideoFrameType::kVideoFrameDelta); + frame_types[2] = VideoFrameType::kVideoFrameKey; + ExpectStreams(VideoFrameType::kVideoFrameKey, kNumberOfSimulcastStreams); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + std::fill(frame_types.begin(), frame_types.end(), + VideoFrameType::kVideoFrameDelta); + ExpectStreams(VideoFrameType::kVideoFrameDelta, kNumberOfSimulcastStreams); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); +} + +// For some codecs (H264) expect only particular active streams to generate a +// key frame when a key frame was only requested for some of them. +void SimulcastTestFixtureImpl::TestKeyFrameRequestsOnSpecificStreams() { + SetRates(kMaxBitrates[2], 30); // To get all three streams. + std::vector<VideoFrameType> frame_types(kNumberOfSimulcastStreams, + VideoFrameType::kVideoFrameDelta); + ExpectStreams(VideoFrameType::kVideoFrameKey, kNumberOfSimulcastStreams); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + ExpectStreams(VideoFrameType::kVideoFrameDelta, kNumberOfSimulcastStreams); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + frame_types[0] = VideoFrameType::kVideoFrameKey; + ExpectStream(VideoFrameType::kVideoFrameKey, kScaleResolutionDownBy[0]); + ExpectStream(VideoFrameType::kVideoFrameDelta, kScaleResolutionDownBy[1]); + ExpectStream(VideoFrameType::kVideoFrameDelta, kScaleResolutionDownBy[2]); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + std::fill(frame_types.begin(), frame_types.end(), + VideoFrameType::kVideoFrameDelta); + frame_types[1] = VideoFrameType::kVideoFrameKey; + ExpectStream(VideoFrameType::kVideoFrameDelta, kScaleResolutionDownBy[0]); + ExpectStream(VideoFrameType::kVideoFrameKey, kScaleResolutionDownBy[1]); + ExpectStream(VideoFrameType::kVideoFrameDelta, kScaleResolutionDownBy[2]); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + std::fill(frame_types.begin(), frame_types.end(), + VideoFrameType::kVideoFrameDelta); + frame_types[2] = VideoFrameType::kVideoFrameKey; + ExpectStream(VideoFrameType::kVideoFrameDelta, kScaleResolutionDownBy[0]); + ExpectStream(VideoFrameType::kVideoFrameDelta, kScaleResolutionDownBy[1]); + ExpectStream(VideoFrameType::kVideoFrameKey, kScaleResolutionDownBy[2]); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + std::fill(frame_types.begin(), frame_types.end(), + VideoFrameType::kVideoFrameDelta); + frame_types[0] = VideoFrameType::kVideoFrameKey; + frame_types[2] = VideoFrameType::kVideoFrameKey; + ExpectStream(VideoFrameType::kVideoFrameKey, kScaleResolutionDownBy[0]); + ExpectStream(VideoFrameType::kVideoFrameDelta, kScaleResolutionDownBy[1]); + ExpectStream(VideoFrameType::kVideoFrameKey, kScaleResolutionDownBy[2]); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + std::fill(frame_types.begin(), frame_types.end(), + VideoFrameType::kVideoFrameKey); + ExpectStream(VideoFrameType::kVideoFrameKey, kScaleResolutionDownBy[0]); + ExpectStream(VideoFrameType::kVideoFrameKey, kScaleResolutionDownBy[1]); + ExpectStream(VideoFrameType::kVideoFrameKey, kScaleResolutionDownBy[2]); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + std::fill(frame_types.begin(), frame_types.end(), + VideoFrameType::kVideoFrameDelta); + ExpectStreams(VideoFrameType::kVideoFrameDelta, kNumberOfSimulcastStreams); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); +} + +void SimulcastTestFixtureImpl::TestPaddingAllStreams() { + // We should always encode the base layer. + SetRates(kMinBitrates[0] - 1, 30); + std::vector<VideoFrameType> frame_types(kNumberOfSimulcastStreams, + VideoFrameType::kVideoFrameDelta); + ExpectStreams(VideoFrameType::kVideoFrameKey, 1); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + ExpectStreams(VideoFrameType::kVideoFrameDelta, 1); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); +} + +void SimulcastTestFixtureImpl::TestPaddingTwoStreams() { + // We have just enough to get only the first stream and padding for two. + SetRates(kMinBitrates[0], 30); + std::vector<VideoFrameType> frame_types(kNumberOfSimulcastStreams, + VideoFrameType::kVideoFrameDelta); + ExpectStreams(VideoFrameType::kVideoFrameKey, 1); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + ExpectStreams(VideoFrameType::kVideoFrameDelta, 1); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); +} + +void SimulcastTestFixtureImpl::TestPaddingTwoStreamsOneMaxedOut() { + // We are just below limit of sending second stream, so we should get + // the first stream maxed out (at `maxBitrate`), and padding for two. + SetRates(kTargetBitrates[0] + kMinBitrates[1] - 1, 30); + std::vector<VideoFrameType> frame_types(kNumberOfSimulcastStreams, + VideoFrameType::kVideoFrameDelta); + ExpectStreams(VideoFrameType::kVideoFrameKey, 1); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + ExpectStreams(VideoFrameType::kVideoFrameDelta, 1); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); +} + +void SimulcastTestFixtureImpl::TestPaddingOneStream() { + // We have just enough to send two streams, so padding for one stream. + SetRates(kTargetBitrates[0] + kMinBitrates[1], 30); + std::vector<VideoFrameType> frame_types(kNumberOfSimulcastStreams, + VideoFrameType::kVideoFrameDelta); + ExpectStreams(VideoFrameType::kVideoFrameKey, 2); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + ExpectStreams(VideoFrameType::kVideoFrameDelta, 2); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); +} + +void SimulcastTestFixtureImpl::TestPaddingOneStreamTwoMaxedOut() { + // We are just below limit of sending third stream, so we should get + // first stream's rate maxed out at `targetBitrate`, second at `maxBitrate`. + SetRates(kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2] - 1, 30); + std::vector<VideoFrameType> frame_types(kNumberOfSimulcastStreams, + VideoFrameType::kVideoFrameDelta); + ExpectStreams(VideoFrameType::kVideoFrameKey, 2); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + ExpectStreams(VideoFrameType::kVideoFrameDelta, 2); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); +} + +void SimulcastTestFixtureImpl::TestSendAllStreams() { + // We have just enough to send all streams. + SetRates(kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2], 30); + std::vector<VideoFrameType> frame_types(kNumberOfSimulcastStreams, + VideoFrameType::kVideoFrameDelta); + ExpectStreams(VideoFrameType::kVideoFrameKey, 3); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + ExpectStreams(VideoFrameType::kVideoFrameDelta, 3); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); +} + +void SimulcastTestFixtureImpl::TestDisablingStreams() { + // We should get three media streams. + SetRates(kMaxBitrates[0] + kMaxBitrates[1] + kMaxBitrates[2], 30); + std::vector<VideoFrameType> frame_types(kNumberOfSimulcastStreams, + VideoFrameType::kVideoFrameDelta); + ExpectStreams(VideoFrameType::kVideoFrameKey, 3); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + ExpectStreams(VideoFrameType::kVideoFrameDelta, 3); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + // We should only get two streams and padding for one. + SetRates(kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2] / 2, 30); + ExpectStreams(VideoFrameType::kVideoFrameDelta, 2); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + // We should only get the first stream and padding for two. + SetRates(kTargetBitrates[0] + kMinBitrates[1] / 2, 30); + ExpectStreams(VideoFrameType::kVideoFrameDelta, 1); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + // We don't have enough bitrate for the thumbnail stream, but we should get + // it anyway with current configuration. + SetRates(kTargetBitrates[0] - 1, 30); + ExpectStreams(VideoFrameType::kVideoFrameDelta, 1); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + // We should only get two streams and padding for one. + SetRates(kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2] / 2, 30); + // We get a key frame because a new stream is being enabled. + ExpectStreams(VideoFrameType::kVideoFrameKey, 2); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + // We should get all three streams. + SetRates(kTargetBitrates[0] + kTargetBitrates[1] + kTargetBitrates[2], 30); + // We get a key frame because a new stream is being enabled. + ExpectStreams(VideoFrameType::kVideoFrameKey, 3); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); +} + +void SimulcastTestFixtureImpl::TestActiveStreams() { + // All streams on. + RunActiveStreamsTest({true, true, true}); + // All streams off. + RunActiveStreamsTest({false, false, false}); + // Low stream off. + RunActiveStreamsTest({false, true, true}); + // Middle stream off. + RunActiveStreamsTest({true, false, true}); + // High stream off. + RunActiveStreamsTest({true, true, false}); + // Only low stream turned on. + RunActiveStreamsTest({true, false, false}); + // Only middle stream turned on. + RunActiveStreamsTest({false, true, false}); + // Only high stream turned on. + RunActiveStreamsTest({false, false, true}); +} + +void SimulcastTestFixtureImpl::SwitchingToOneStream(int width, int height) { + const int* temporal_layer_profile = nullptr; + // Disable all streams except the last and set the bitrate of the last to + // 100 kbps. This verifies the way GTP switches to screenshare mode. + if (codec_type_ == kVideoCodecVP8) { + settings_.VP8()->numberOfTemporalLayers = 1; + temporal_layer_profile = kDefaultTemporalLayerProfile; + } else { + settings_.H264()->numberOfTemporalLayers = 1; + temporal_layer_profile = kNoTemporalLayerProfile; + } + settings_.maxBitrate = 100; + settings_.startBitrate = 100; + settings_.width = width; + settings_.height = height; + for (int i = 0; i < settings_.numberOfSimulcastStreams - 1; ++i) { + settings_.simulcastStream[i].maxBitrate = 0; + settings_.simulcastStream[i].width = settings_.width; + settings_.simulcastStream[i].height = settings_.height; + settings_.simulcastStream[i].numberOfTemporalLayers = 1; + } + // Setting input image to new resolution. + input_buffer_ = I420Buffer::Create(settings_.width, settings_.height); + input_buffer_->InitializeData(); + + input_frame_ = std::make_unique<webrtc::VideoFrame>( + webrtc::VideoFrame::Builder() + .set_video_frame_buffer(input_buffer_) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build()); + + // The for loop above did not set the bitrate of the highest layer. + settings_.simulcastStream[settings_.numberOfSimulcastStreams - 1].maxBitrate = + 0; + // The highest layer has to correspond to the non-simulcast resolution. + settings_.simulcastStream[settings_.numberOfSimulcastStreams - 1].width = + settings_.width; + settings_.simulcastStream[settings_.numberOfSimulcastStreams - 1].height = + settings_.height; + SetUpRateAllocator(); + EXPECT_EQ(0, encoder_->InitEncode(&settings_, kSettings)); + + // Encode one frame and verify. + SetRates(kMaxBitrates[0] + kMaxBitrates[1], 30); + std::vector<VideoFrameType> frame_types(kNumberOfSimulcastStreams, + VideoFrameType::kVideoFrameDelta); + EXPECT_CALL( + encoder_callback_, + OnEncodedImage(AllOf(Field(&EncodedImage::_frameType, + VideoFrameType::kVideoFrameKey), + Field(&EncodedImage::_encodedWidth, width), + Field(&EncodedImage::_encodedHeight, height)), + _)) + .Times(1) + .WillRepeatedly(Return( + EncodedImageCallback::Result(EncodedImageCallback::Result::OK, 0))); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); + + // Switch back. + DefaultSettings(&settings_, temporal_layer_profile, codec_type_); + // Start at the lowest bitrate for enabling base stream. + settings_.startBitrate = kMinBitrates[0]; + SetUpRateAllocator(); + EXPECT_EQ(0, encoder_->InitEncode(&settings_, kSettings)); + SetRates(settings_.startBitrate, 30); + ExpectStreams(VideoFrameType::kVideoFrameKey, 1); + // Resize `input_frame_` to the new resolution. + input_buffer_ = I420Buffer::Create(settings_.width, settings_.height); + input_buffer_->InitializeData(); + input_frame_ = std::make_unique<webrtc::VideoFrame>( + webrtc::VideoFrame::Builder() + .set_video_frame_buffer(input_buffer_) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build()); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, &frame_types)); +} + +void SimulcastTestFixtureImpl::TestSwitchingToOneStream() { + SwitchingToOneStream(1024, 768); +} + +void SimulcastTestFixtureImpl::TestSwitchingToOneOddStream() { + SwitchingToOneStream(1023, 769); +} + +void SimulcastTestFixtureImpl::TestSwitchingToOneSmallStream() { + SwitchingToOneStream(4, 4); +} + +// Test the layer pattern and sync flag for various spatial-temporal patterns. +// 3-3-3 pattern: 3 temporal layers for all spatial streams, so same +// temporal_layer id and layer_sync is expected for all streams. +void SimulcastTestFixtureImpl::TestSpatioTemporalLayers333PatternEncoder() { + bool is_h264 = codec_type_ == kVideoCodecH264; + TestEncodedImageCallback encoder_callback; + encoder_->RegisterEncodeCompleteCallback(&encoder_callback); + SetRates(kMaxBitrates[2], 30); // To get all three streams. + + int expected_temporal_idx[3] = {-1, -1, -1}; + bool expected_layer_sync[3] = {false, false, false}; + + // First frame: #0. + EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL)); + SetExpectedValues3<int>(0, 0, 0, expected_temporal_idx); + SetExpectedValues3<bool>(!is_h264, !is_h264, !is_h264, expected_layer_sync); + VerifyTemporalIdxAndSyncForAllSpatialLayers( + &encoder_callback, expected_temporal_idx, expected_layer_sync, 3); + + // Next frame: #1. + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL)); + SetExpectedValues3<int>(2, 2, 2, expected_temporal_idx); + SetExpectedValues3<bool>(true, true, true, expected_layer_sync); + VerifyTemporalIdxAndSyncForAllSpatialLayers( + &encoder_callback, expected_temporal_idx, expected_layer_sync, 3); + + // Next frame: #2. + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL)); + SetExpectedValues3<int>(1, 1, 1, expected_temporal_idx); + SetExpectedValues3<bool>(true, true, true, expected_layer_sync); + VerifyTemporalIdxAndSyncForAllSpatialLayers( + &encoder_callback, expected_temporal_idx, expected_layer_sync, 3); + + // Next frame: #3. + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL)); + SetExpectedValues3<int>(2, 2, 2, expected_temporal_idx); + SetExpectedValues3<bool>(false, false, false, expected_layer_sync); + VerifyTemporalIdxAndSyncForAllSpatialLayers( + &encoder_callback, expected_temporal_idx, expected_layer_sync, 3); + + // Next frame: #4. + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL)); + SetExpectedValues3<int>(0, 0, 0, expected_temporal_idx); + SetExpectedValues3<bool>(false, false, false, expected_layer_sync); + VerifyTemporalIdxAndSyncForAllSpatialLayers( + &encoder_callback, expected_temporal_idx, expected_layer_sync, 3); + + // Next frame: #5. + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL)); + SetExpectedValues3<int>(2, 2, 2, expected_temporal_idx); + SetExpectedValues3<bool>(is_h264, is_h264, is_h264, expected_layer_sync); + VerifyTemporalIdxAndSyncForAllSpatialLayers( + &encoder_callback, expected_temporal_idx, expected_layer_sync, 3); +} + +// Test the layer pattern and sync flag for various spatial-temporal patterns. +// 3-2-1 pattern: 3 temporal layers for lowest resolution, 2 for middle, and +// 1 temporal layer for highest resolution. +// For this profile, we expect the temporal index pattern to be: +// 1st stream: 0, 2, 1, 2, .... +// 2nd stream: 0, 1, 0, 1, ... +// 3rd stream: -1, -1, -1, -1, .... +// Regarding the 3rd stream, note that a stream/encoder with 1 temporal layer +// should always have temporal layer idx set to kNoTemporalIdx = -1. +// Since CodecSpecificInfoVP8.temporalIdx is uint8_t, this will wrap to 255. +// TODO(marpan): Although this seems safe for now, we should fix this. +void SimulcastTestFixtureImpl::TestSpatioTemporalLayers321PatternEncoder() { + EXPECT_EQ(codec_type_, kVideoCodecVP8); + int temporal_layer_profile[3] = {3, 2, 1}; + SetUpCodec(temporal_layer_profile); + TestEncodedImageCallback encoder_callback; + encoder_->RegisterEncodeCompleteCallback(&encoder_callback); + SetRates(kMaxBitrates[2], 30); // To get all three streams. + + int expected_temporal_idx[3] = {-1, -1, -1}; + bool expected_layer_sync[3] = {false, false, false}; + + // First frame: #0. + EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL)); + SetExpectedValues3<int>(0, 0, 255, expected_temporal_idx); + SetExpectedValues3<bool>(true, true, false, expected_layer_sync); + VerifyTemporalIdxAndSyncForAllSpatialLayers( + &encoder_callback, expected_temporal_idx, expected_layer_sync, 3); + + // Next frame: #1. + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL)); + SetExpectedValues3<int>(2, 1, 255, expected_temporal_idx); + SetExpectedValues3<bool>(true, true, false, expected_layer_sync); + VerifyTemporalIdxAndSyncForAllSpatialLayers( + &encoder_callback, expected_temporal_idx, expected_layer_sync, 3); + + // Next frame: #2. + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL)); + SetExpectedValues3<int>(1, 0, 255, expected_temporal_idx); + SetExpectedValues3<bool>(true, false, false, expected_layer_sync); + VerifyTemporalIdxAndSyncForAllSpatialLayers( + &encoder_callback, expected_temporal_idx, expected_layer_sync, 3); + + // Next frame: #3. + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL)); + SetExpectedValues3<int>(2, 1, 255, expected_temporal_idx); + SetExpectedValues3<bool>(false, false, false, expected_layer_sync); + VerifyTemporalIdxAndSyncForAllSpatialLayers( + &encoder_callback, expected_temporal_idx, expected_layer_sync, 3); + + // Next frame: #4. + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL)); + SetExpectedValues3<int>(0, 0, 255, expected_temporal_idx); + SetExpectedValues3<bool>(false, false, false, expected_layer_sync); + VerifyTemporalIdxAndSyncForAllSpatialLayers( + &encoder_callback, expected_temporal_idx, expected_layer_sync, 3); + + // Next frame: #5. + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL)); + SetExpectedValues3<int>(2, 1, 255, expected_temporal_idx); + SetExpectedValues3<bool>(false, true, false, expected_layer_sync); + VerifyTemporalIdxAndSyncForAllSpatialLayers( + &encoder_callback, expected_temporal_idx, expected_layer_sync, 3); +} + +void SimulcastTestFixtureImpl::TestStrideEncodeDecode() { + TestEncodedImageCallback encoder_callback; + TestDecodedImageCallback decoder_callback; + encoder_->RegisterEncodeCompleteCallback(&encoder_callback); + decoder_->RegisterDecodeCompleteCallback(&decoder_callback); + + SetRates(kMaxBitrates[2], 30); // To get all three streams. + // Setting two (possibly) problematic use cases for stride: + // 1. stride > width 2. stride_y != stride_uv/2 + int stride_y = kDefaultWidth + 20; + int stride_uv = ((kDefaultWidth + 1) / 2) + 5; + input_buffer_ = I420Buffer::Create(kDefaultWidth, kDefaultHeight, stride_y, + stride_uv, stride_uv); + input_frame_ = std::make_unique<webrtc::VideoFrame>( + webrtc::VideoFrame::Builder() + .set_video_frame_buffer(input_buffer_) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build()); + + // Set color. + int plane_offset[kNumOfPlanes]; + plane_offset[kYPlane] = kColorY; + plane_offset[kUPlane] = kColorU; + plane_offset[kVPlane] = kColorV; + CreateImage(input_buffer_, plane_offset); + + EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL)); + + // Change color. + plane_offset[kYPlane] += 1; + plane_offset[kUPlane] += 1; + plane_offset[kVPlane] += 1; + CreateImage(input_buffer_, plane_offset); + input_frame_->set_timestamp(input_frame_->timestamp() + 3000); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL)); + + EncodedImage encoded_frame; + // Only encoding one frame - so will be a key frame. + encoder_callback.GetLastEncodedKeyFrame(&encoded_frame); + EXPECT_EQ(0, decoder_->Decode(encoded_frame, 0)); + encoder_callback.GetLastEncodedFrame(&encoded_frame); + decoder_->Decode(encoded_frame, 0); + EXPECT_EQ(2, decoder_callback.DecodedFrames()); +} + +void SimulcastTestFixtureImpl::TestDecodeWidthHeightSet() { + MockEncodedImageCallback encoder_callback; + MockDecodedImageCallback decoder_callback; + + EncodedImage encoded_frame[3]; + SetRates(kMaxBitrates[2], 30); // To get all three streams. + encoder_->RegisterEncodeCompleteCallback(&encoder_callback); + decoder_->RegisterDecodeCompleteCallback(&decoder_callback); + + EXPECT_CALL(encoder_callback, OnEncodedImage(_, _)) + .Times(3) + .WillRepeatedly( + ::testing::Invoke([&](const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) { + EXPECT_EQ(encoded_image._frameType, VideoFrameType::kVideoFrameKey); + + size_t index = encoded_image.SimulcastIndex().value_or(0); + encoded_frame[index].SetEncodedData(EncodedImageBuffer::Create( + encoded_image.data(), encoded_image.size())); + encoded_frame[index]._frameType = encoded_image._frameType; + return EncodedImageCallback::Result( + EncodedImageCallback::Result::OK, 0); + })); + EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL)); + + EXPECT_CALL(decoder_callback, Decoded(_, _, _)) + .WillOnce(::testing::Invoke([](VideoFrame& decodedImage, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp) { + EXPECT_EQ(decodedImage.width(), kDefaultWidth / 4); + EXPECT_EQ(decodedImage.height(), kDefaultHeight / 4); + })); + EXPECT_EQ(0, decoder_->Decode(encoded_frame[0], 0)); + + EXPECT_CALL(decoder_callback, Decoded(_, _, _)) + .WillOnce(::testing::Invoke([](VideoFrame& decodedImage, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp) { + EXPECT_EQ(decodedImage.width(), kDefaultWidth / 2); + EXPECT_EQ(decodedImage.height(), kDefaultHeight / 2); + })); + EXPECT_EQ(0, decoder_->Decode(encoded_frame[1], 0)); + + EXPECT_CALL(decoder_callback, Decoded(_, _, _)) + .WillOnce(::testing::Invoke([](VideoFrame& decodedImage, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp) { + EXPECT_EQ(decodedImage.width(), kDefaultWidth); + EXPECT_EQ(decodedImage.height(), kDefaultHeight); + })); + EXPECT_EQ(0, decoder_->Decode(encoded_frame[2], 0)); +} + +void SimulcastTestFixtureImpl:: + TestEncoderInfoForDefaultTemporalLayerProfileHasFpsAllocation() { + VideoEncoder::EncoderInfo encoder_info = encoder_->GetEncoderInfo(); + EXPECT_EQ(encoder_info.fps_allocation[0].size(), + static_cast<size_t>(kDefaultTemporalLayerProfile[0])); + EXPECT_EQ(encoder_info.fps_allocation[1].size(), + static_cast<size_t>(kDefaultTemporalLayerProfile[1])); + EXPECT_EQ(encoder_info.fps_allocation[2].size(), + static_cast<size_t>(kDefaultTemporalLayerProfile[2])); +} +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/simulcast_test_fixture_impl.h b/third_party/libwebrtc/modules/video_coding/utility/simulcast_test_fixture_impl.h new file mode 100644 index 0000000000..f142ab4813 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/simulcast_test_fixture_impl.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_SIMULCAST_TEST_FIXTURE_IMPL_H_ +#define MODULES_VIDEO_CODING_UTILITY_SIMULCAST_TEST_FIXTURE_IMPL_H_ + +#include <memory> +#include <vector> + +#include "api/test/mock_video_decoder.h" +#include "api/test/mock_video_encoder.h" +#include "api/test/simulcast_test_fixture.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/video_coding/utility/simulcast_rate_allocator.h" + +namespace webrtc { +namespace test { + +class SimulcastTestFixtureImpl final : public SimulcastTestFixture { + public: + SimulcastTestFixtureImpl(std::unique_ptr<VideoEncoderFactory> encoder_factory, + std::unique_ptr<VideoDecoderFactory> decoder_factory, + SdpVideoFormat video_format); + ~SimulcastTestFixtureImpl() final; + + // Implements SimulcastTestFixture. + void TestKeyFrameRequestsOnAllStreams() override; + void TestKeyFrameRequestsOnSpecificStreams() override; + void TestPaddingAllStreams() override; + void TestPaddingTwoStreams() override; + void TestPaddingTwoStreamsOneMaxedOut() override; + void TestPaddingOneStream() override; + void TestPaddingOneStreamTwoMaxedOut() override; + void TestSendAllStreams() override; + void TestDisablingStreams() override; + void TestActiveStreams() override; + void TestSwitchingToOneStream() override; + void TestSwitchingToOneOddStream() override; + void TestSwitchingToOneSmallStream() override; + void TestSpatioTemporalLayers333PatternEncoder() override; + void TestSpatioTemporalLayers321PatternEncoder() override; + void TestStrideEncodeDecode() override; + void TestDecodeWidthHeightSet() override; + void TestEncoderInfoForDefaultTemporalLayerProfileHasFpsAllocation() override; + + static void DefaultSettings(VideoCodec* settings, + const int* temporal_layer_profile, + VideoCodecType codec_type, + bool reverse_layer_order = false); + + private: + class TestEncodedImageCallback; + class TestDecodedImageCallback; + + void SetUpCodec(const int* temporal_layer_profile); + void SetUpRateAllocator(); + void SetRates(uint32_t bitrate_kbps, uint32_t fps); + void RunActiveStreamsTest(std::vector<bool> active_streams); + void UpdateActiveStreams(std::vector<bool> active_streams); + void ExpectStream(VideoFrameType frame_type, int scaleResolutionDownBy); + void ExpectStreams(VideoFrameType frame_type, + std::vector<bool> expected_streams_active); + void ExpectStreams(VideoFrameType frame_type, int expected_video_streams); + void VerifyTemporalIdxAndSyncForAllSpatialLayers( + TestEncodedImageCallback* encoder_callback, + const int* expected_temporal_idx, + const bool* expected_layer_sync, + int num_spatial_layers); + void SwitchingToOneStream(int width, int height); + + std::unique_ptr<VideoEncoder> encoder_; + MockEncodedImageCallback encoder_callback_; + std::unique_ptr<VideoDecoder> decoder_; + MockDecodedImageCallback decoder_callback_; + VideoCodec settings_; + rtc::scoped_refptr<I420Buffer> input_buffer_; + std::unique_ptr<VideoFrame> input_frame_; + std::unique_ptr<SimulcastRateAllocator> rate_allocator_; + VideoCodecType codec_type_; +}; + +} // namespace test +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_UTILITY_SIMULCAST_TEST_FIXTURE_IMPL_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/simulcast_utility.cc b/third_party/libwebrtc/modules/video_coding/utility/simulcast_utility.cc new file mode 100644 index 0000000000..824f4b0eac --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/simulcast_utility.cc @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/simulcast_utility.h" + +#include <algorithm> +#include <cmath> + +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +uint32_t SimulcastUtility::SumStreamMaxBitrate(int streams, + const VideoCodec& codec) { + uint32_t bitrate_sum = 0; + for (int i = 0; i < streams; ++i) { + bitrate_sum += codec.simulcastStream[i].maxBitrate; + } + return bitrate_sum; +} + +int SimulcastUtility::NumberOfSimulcastStreams(const VideoCodec& codec) { + int streams = + codec.numberOfSimulcastStreams < 1 ? 1 : codec.numberOfSimulcastStreams; + uint32_t simulcast_max_bitrate = SumStreamMaxBitrate(streams, codec); + if (simulcast_max_bitrate == 0) { + streams = 1; + } + return streams; +} + +bool SimulcastUtility::ValidSimulcastParameters(const VideoCodec& codec, + int num_streams) { + // Check resolution. + if (codec.width != codec.simulcastStream[num_streams - 1].width || + codec.height != codec.simulcastStream[num_streams - 1].height) { + return false; + } + for (int i = 0; i < num_streams; ++i) { + if (codec.width * codec.simulcastStream[i].height != + codec.height * codec.simulcastStream[i].width) { + return false; + } + } + for (int i = 1; i < num_streams; ++i) { + if (codec.simulcastStream[i].width < codec.simulcastStream[i - 1].width) { + return false; + } + } + + // Check frame-rate. + for (int i = 1; i < num_streams; ++i) { + if (fabs(codec.simulcastStream[i].maxFramerate - + codec.simulcastStream[i - 1].maxFramerate) > 1e-9) { + return false; + } + } + + // Check temporal layers. + for (int i = 0; i < num_streams - 1; ++i) { + if (codec.simulcastStream[i].numberOfTemporalLayers != + codec.simulcastStream[i + 1].numberOfTemporalLayers) + return false; + } + return true; +} + +bool SimulcastUtility::IsConferenceModeScreenshare(const VideoCodec& codec) { + return codec.mode == VideoCodecMode::kScreensharing && + codec.legacy_conference_mode; +} + +int SimulcastUtility::NumberOfTemporalLayers(const VideoCodec& codec, + int spatial_id) { + int num_temporal_layers = 0; + if (auto scalability_mode = codec.GetScalabilityMode(); scalability_mode) { + num_temporal_layers = ScalabilityModeToNumTemporalLayers(*scalability_mode); + } else { + switch (codec.codecType) { + case kVideoCodecVP8: + num_temporal_layers = codec.VP8().numberOfTemporalLayers; + break; + case kVideoCodecVP9: + num_temporal_layers = codec.VP9().numberOfTemporalLayers; + break; + case kVideoCodecH264: + num_temporal_layers = codec.H264().numberOfTemporalLayers; + break; + case kVideoCodecH265: + // TODO(bugs.webrtc.org/13485) + break; + default: + break; + } + } + + if (codec.numberOfSimulcastStreams > 0) { + RTC_DCHECK_LT(spatial_id, codec.numberOfSimulcastStreams); + num_temporal_layers = + std::max(num_temporal_layers, + static_cast<int>( + codec.simulcastStream[spatial_id].numberOfTemporalLayers)); + } + return std::max(1, num_temporal_layers); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/simulcast_utility.h b/third_party/libwebrtc/modules/video_coding/utility/simulcast_utility.h new file mode 100644 index 0000000000..e25a594360 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/simulcast_utility.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_SIMULCAST_UTILITY_H_ +#define MODULES_VIDEO_CODING_UTILITY_SIMULCAST_UTILITY_H_ + +#include <stdint.h> + +#include "api/video_codecs/video_codec.h" + +namespace webrtc { + +class SimulcastUtility { + public: + static uint32_t SumStreamMaxBitrate(int streams, const VideoCodec& codec); + static int NumberOfSimulcastStreams(const VideoCodec& codec); + static bool ValidSimulcastParameters(const VideoCodec& codec, + int num_streams); + static int NumberOfTemporalLayers(const VideoCodec& codec, int spatial_id); + // TODO(sprang): Remove this hack when ScreenshareLayers is gone. + static bool IsConferenceModeScreenshare(const VideoCodec& codec); +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_UTILITY_SIMULCAST_UTILITY_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/vp8_constants.h b/third_party/libwebrtc/modules/video_coding/utility/vp8_constants.h new file mode 100644 index 0000000000..9321864dbc --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/vp8_constants.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_VP8_CONSTANTS_H_ +#define MODULES_VIDEO_CODING_UTILITY_VP8_CONSTANTS_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <string> + +namespace webrtc { + +// QP level below which VP8 variable framerate and zero hertz screencast reduces +// framerate due to diminishing quality enhancement returns. +constexpr int kVp8SteadyStateQpThreshold = 15; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_UTILITY_VP8_CONSTANTS_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/vp8_header_parser.cc b/third_party/libwebrtc/modules/video_coding/utility/vp8_header_parser.cc new file mode 100644 index 0000000000..80026f9a0f --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/vp8_header_parser.cc @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/utility/vp8_header_parser.h" + +#include "rtc_base/logging.h" +#include "rtc_base/system/arch.h" + +namespace webrtc { + +namespace vp8 { +namespace { +const size_t kCommonPayloadHeaderLength = 3; +const size_t kKeyPayloadHeaderLength = 10; +const int kMbFeatureTreeProbs = 3; +const int kNumMbSegments = 4; +const int kNumRefLfDeltas = 4; +const int kNumModeLfDeltas = 4; + +} // namespace + +// Bitstream parser according to +// https://tools.ietf.org/html/rfc6386#section-7.3 +void VP8InitBitReader(VP8BitReader* const br, + const uint8_t* start, + const uint8_t* end) { + br->range_ = 255; + br->buf_ = start; + br->buf_end_ = end; + br->value_ = 0; + br->bits_ = 0; + + // Read 2 bytes. + int i = 0; + while (++i <= 2) { + if (br->buf_ != br->buf_end_) { + br->value_ = br->value_ << 8 | *br->buf_++; + } else { + br->value_ = br->value_ << 8; + } + } +} + +// Bit decoder according to https://tools.ietf.org/html/rfc6386#section-7.3 +// Reads one bit from the bitstream, given that it has probability prob/256 to +// be 1. +int Vp8BitReaderGetBool(VP8BitReader* br, int prob) { + uint32_t split = 1 + (((br->range_ - 1) * prob) >> 8); + uint32_t split_hi = split << 8; + int retval = 0; + if (br->value_ >= split_hi) { + retval = 1; + br->range_ -= split; + br->value_ -= split_hi; + } else { + retval = 0; + br->range_ = split; + } + + while (br->range_ < 128) { + br->value_ <<= 1; + br->range_ <<= 1; + if (++br->bits_ == 8) { + br->bits_ = 0; + if (br->buf_ != br->buf_end_) { + br->value_ |= *br->buf_++; + } + } + } + return retval; +} + +uint32_t VP8GetValue(VP8BitReader* br, int num_bits) { + uint32_t v = 0; + while (num_bits--) { + // According to https://tools.ietf.org/html/rfc6386 + // Probability 128/256 is used to encode header fields. + v = (v << 1) | Vp8BitReaderGetBool(br, 128); + } + return v; +} + +// Not a read_signed_literal() from RFC 6386! +// This one is used to read e.g. quantizer_update, which is written as: +// L(num_bits), sign-bit. +int32_t VP8GetSignedValue(VP8BitReader* br, int num_bits) { + int v = VP8GetValue(br, num_bits); + int sign = VP8GetValue(br, 1); + return sign ? -v : v; +} + +static void ParseSegmentHeader(VP8BitReader* br) { + int use_segment = VP8GetValue(br, 1); + if (use_segment) { + int update_map = VP8GetValue(br, 1); + if (VP8GetValue(br, 1)) { // update_segment_feature_data. + VP8GetValue(br, 1); // segment_feature_mode. + int s; + for (s = 0; s < kNumMbSegments; ++s) { + bool quantizer_update = VP8GetValue(br, 1); + if (quantizer_update) { + VP8GetSignedValue(br, 7); + } + } + for (s = 0; s < kNumMbSegments; ++s) { + bool loop_filter_update = VP8GetValue(br, 1); + if (loop_filter_update) { + VP8GetSignedValue(br, 6); + } + } + } + if (update_map) { + int s; + for (s = 0; s < kMbFeatureTreeProbs; ++s) { + bool segment_prob_update = VP8GetValue(br, 1); + if (segment_prob_update) { + VP8GetValue(br, 8); + } + } + } + } +} + +static void ParseFilterHeader(VP8BitReader* br) { + VP8GetValue(br, 1); // filter_type. + VP8GetValue(br, 6); // loop_filter_level. + VP8GetValue(br, 3); // sharpness_level. + + // mb_lf_adjustments. + int loop_filter_adj_enable = VP8GetValue(br, 1); + if (loop_filter_adj_enable) { + int mode_ref_lf_delta_update = VP8GetValue(br, 1); + if (mode_ref_lf_delta_update) { + int i; + for (i = 0; i < kNumRefLfDeltas; ++i) { + int ref_frame_delta_update_flag = VP8GetValue(br, 1); + if (ref_frame_delta_update_flag) { + VP8GetSignedValue(br, 6); // delta_magnitude. + } + } + for (i = 0; i < kNumModeLfDeltas; ++i) { + int mb_mode_delta_update_flag = VP8GetValue(br, 1); + if (mb_mode_delta_update_flag) { + VP8GetSignedValue(br, 6); // delta_magnitude. + } + } + } + } +} + +bool GetQp(const uint8_t* buf, size_t length, int* qp) { + if (length < kCommonPayloadHeaderLength) { + RTC_LOG(LS_WARNING) << "Failed to get QP, invalid length."; + return false; + } + VP8BitReader br; + const uint32_t bits = buf[0] | (buf[1] << 8) | (buf[2] << 16); + int key_frame = !(bits & 1); + // Size of first partition in bytes. + uint32_t partition_length = (bits >> 5); + size_t header_length = kCommonPayloadHeaderLength; + if (key_frame) { + header_length = kKeyPayloadHeaderLength; + } + if (header_length + partition_length > length) { + RTC_LOG(LS_WARNING) << "Failed to get QP, invalid length: " << length; + return false; + } + buf += header_length; + + VP8InitBitReader(&br, buf, buf + partition_length); + if (key_frame) { + // Color space and pixel type. + VP8GetValue(&br, 1); + VP8GetValue(&br, 1); + } + ParseSegmentHeader(&br); + ParseFilterHeader(&br); + // Parse log2_nbr_of_dct_partitions value. + VP8GetValue(&br, 2); + // Base QP. + const int base_q0 = VP8GetValue(&br, 7); + if (br.buf_ == br.buf_end_) { + RTC_LOG(LS_WARNING) << "Failed to get QP, bitstream is truncated or" + " corrupted."; + return false; + } + *qp = base_q0; + return true; +} + +} // namespace vp8 + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/vp8_header_parser.h b/third_party/libwebrtc/modules/video_coding/utility/vp8_header_parser.h new file mode 100644 index 0000000000..dbad999dc8 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/vp8_header_parser.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_VP8_HEADER_PARSER_H_ +#define MODULES_VIDEO_CODING_UTILITY_VP8_HEADER_PARSER_H_ + +#include <stdint.h> +#include <stdio.h> + +namespace webrtc { + +namespace vp8 { + +typedef struct VP8BitReader VP8BitReader; +struct VP8BitReader { + // Boolean decoder. + uint32_t value_; // Current value (2 bytes). + uint32_t range_; // Current range (always in [128..255] interval). + int bits_; // Number of bits shifted out of value, at most 7. + // Read buffer. + const uint8_t* buf_; // Next byte to be read. + const uint8_t* buf_end_; // End of read buffer. +}; + +// Gets the QP, QP range: [0, 127]. +// Returns true on success, false otherwise. +bool GetQp(const uint8_t* buf, size_t length, int* qp); + +} // namespace vp8 + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_UTILITY_VP8_HEADER_PARSER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/vp9_constants.h b/third_party/libwebrtc/modules/video_coding/utility/vp9_constants.h new file mode 100644 index 0000000000..af2c701b82 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/vp9_constants.h @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_VP9_CONSTANTS_H_ +#define MODULES_VIDEO_CODING_UTILITY_VP9_CONSTANTS_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <string> + +namespace webrtc { + +// Number of frames that can be stored for future reference. +constexpr size_t kVp9NumRefFrames = 8; +// Number of frame contexts that can be store for future reference. +constexpr size_t kVp9NumFrameContexts = 4; +// Each inter frame can use up to 3 frames for reference. +constexpr size_t kVp9RefsPerFrame = 3; +// Number of values that can be decoded for mv_fr. +constexpr size_t kVp9MvFrSize = 4; +// Number of positions to search in motion vector prediction. +constexpr size_t kVp9MvrefNeighbours = 8; +// Number of contexts when decoding intra_mode . +constexpr size_t kVp9BlockSizeGroups = 4; +// Number of different block sizes used. +constexpr size_t kVp9BlockSizes = 13; +// Sentinel value to mark partition choices that are illegal. +constexpr size_t kVp9BlockInvalid = 14; +// Number of contexts when decoding partition. +constexpr size_t kVp9PartitionContexts = 16; +// Smallest size of a mode info block. +constexpr size_t kVp9MiSize = 8; +// Minimum width of a tile in units of superblocks (although tiles on +// the right hand edge can be narrower). +constexpr size_t kVp9MinTileWidth_B64 = 4; +// Maximum width of a tile in units of superblocks. +constexpr size_t kVp9MaxTileWidth_B64 = 64; +// Number of motion vectors returned by find_mv_refs process. +constexpr size_t kVp9MaxMvRefCandidates = 2; +// Number of values that can be derived for ref_frame. +constexpr size_t kVp9MaxRefFrames = 4; +// Number of contexts for is_inter. +constexpr size_t kVp9IsInterContexts = 4; +// Number of contexts for comp_mode. +constexpr size_t kVp9CompModeContexts = 5; +// Number of contexts for single_ref and comp_ref. +constexpr size_t kVp9RefContexts = 5; +// Number of segments allowed in segmentation map. +constexpr size_t kVp9MaxSegments = 8; +// Index for quantizer segment feature. +constexpr size_t kVp9SegLvlAlt_Q = 0; +// Index for loop filter segment feature. +constexpr size_t kVp9SegLvlAlt_L = 1; +// Index for reference frame segment feature. +constexpr size_t kVp9SegLvlRefFrame = 2; +// Index for skip segment feature. +constexpr size_t kVp9SegLvlSkip = 3; +// Number of segment features. +constexpr size_t kVp9SegLvlMax = 4; +// Number of different plane types (Y or UV). +constexpr size_t kVp9BlockTypes = 2; +// Number of different prediction types (intra or inter). +constexpr size_t kVp9RefTypes = 2; +// Number of coefficient bands. +constexpr size_t kVp9CoefBands = 6; +// Number of contexts for decoding coefficients. +constexpr size_t kVp9PrevCoefContexts = 6; +// Number of coefficient probabilities that are directly transmitted. +constexpr size_t kVp9UnconstrainedNodes = 3; +// Number of contexts for transform size. +constexpr size_t kVp9TxSizeContexts = 2; +// Number of values for interp_filter. +constexpr size_t kVp9SwitchableFilters = 3; +// Number of contexts for interp_filter. +constexpr size_t kVp9InterpFilterContexts = 4; +// Number of contexts for decoding skip. +constexpr size_t kVp9SkipContexts = 3; +// Number of values for partition. +constexpr size_t kVp9PartitionTypes = 4; +// Number of values for tx_size. +constexpr size_t kVp9TxSizes = 4; +// Number of values for tx_mode. +constexpr size_t kVp9TxModes = 5; +// Inverse transform rows with DCT and columns with DCT. +constexpr size_t kVp9DctDct = 0; +// Inverse transform rows with DCT and columns with ADST. +constexpr size_t kVp9AdstDct = 1; +// Inverse transform rows with ADST and columns with DCT. +constexpr size_t kVp9DctAdst = 2; +// Inverse transform rows with ADST and columns with ADST. +constexpr size_t kVp9AdstAdst = 3; +// Number of values for y_mode. +constexpr size_t kVp9MbModeCount = 14; +// Number of values for intra_mode. +constexpr size_t kVp9IntraModes = 10; +// Number of values for inter_mode. +constexpr size_t kVp9InterModes = 4; +// Number of contexts for inter_mode. +constexpr size_t kVp9InterModeContexts = 7; +// Number of values for mv_joint. +constexpr size_t kVp9MvJoints = 4; +// Number of values for mv_class. +constexpr size_t kVp9MvClasses = 11; +// Number of values for mv_class0_bit. +constexpr size_t kVp9Class0Size = 2; +// Maximum number of bits for decoding motion vectors. +constexpr size_t kVp9MvOffsetBits = 10; +// Number of values allowed for a probability adjustment. +constexpr size_t kVp9MaxProb = 255; +// Number of different mode types for loop filtering. +constexpr size_t kVp9MaxModeLfDeltas = 2; +// Threshold at which motion vectors are considered large. +constexpr size_t kVp9CompandedMvrefThresh = 8; +// Maximum value used for loop filtering. +constexpr size_t kVp9MaxLoopFilter = 63; +// Number of bits of precision when scaling reference frames. +constexpr size_t kVp9RefScaleShift = 14; +// Number of bits of precision when performing inter prediction. +constexpr size_t kVp9SubpelBits = 4; +// 1 << kVp9SubpelBits. +constexpr size_t kVp9SubpelShifts = 16; +// kVp9SubpelShifts - 1. +constexpr size_t kVp9SubpelMask = 15; +// Value used when clipping motion vectors. +constexpr size_t kVp9MvBorder = 128; +// Value used when clipping motion vectors. +constexpr size_t kVp9InterpExtend = 4; +// Value used when clipping motion vectors. +constexpr size_t kVp9Borderinpixels = 160; +// Value used in adapting probabilities. +constexpr size_t kVp9MaxUpdateFactor = 128; +// Value used in adapting probabilities. +constexpr size_t kVp9CountSat = 20; +// Both candidates use ZEROMV. +constexpr size_t kVp9BothZero = 0; +// One candidate uses ZEROMV, one uses NEARMV or NEARESTMV. +constexpr size_t kVp9ZeroPlusPredicted = 1; +// Both candidates use NEARMV or NEARESTMV. +constexpr size_t kVp9BothPredicted = 2; +// One candidate uses NEWMV, one uses ZEROMV. +constexpr size_t kVp9NewPlusNonIntra = 3; +// Both candidates use NEWMV. +constexpr size_t kVp9BothNew = 4; +// One candidate uses intra prediction, one uses inter prediction. +constexpr size_t kVp9IntraPlusNonIntra = 5; +// Both candidates use intra prediction. +constexpr size_t kVp9BothIntra = 6; +// Sentinel value marking a case that can never occur. +constexpr size_t kVp9InvalidCase = 9; + +enum class Vp9TxMode : uint8_t { + kOnly4X4 = 0, + kAllow8X8 = 1, + kAllow16x16 = 2, + kAllow32x32 = 3, + kTxModeSelect = 4 +}; + +enum Vp9BlockSize : uint8_t { + kBlock4X4 = 0, + kBlock4X8 = 1, + kBlock8X4 = 2, + kBlock8X8 = 3, + kBlock8X16 = 4, + kBlock16X8 = 5, + kBlock16X16 = 6, + kBlock16X32 = 7, + kBlock32X16 = 8, + kBlock32X32 = 9, + kBlock32X64 = 10, + kBlock64X32 = 11, + kBlock64X64 = 12 +}; + +enum Vp9Partition : uint8_t { + kPartitionNone = 0, + kPartitionHorizontal = 1, + kPartitionVertical = 2, + kPartitionSplit = 3 +}; + +enum class Vp9ReferenceMode : uint8_t { + kSingleReference = 0, + kCompoundReference = 1, + kReferenceModeSelect = 2, +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_UTILITY_VP9_CONSTANTS_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.cc b/third_party/libwebrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.cc new file mode 100644 index 0000000000..bf9d51f692 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.cc @@ -0,0 +1,533 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" + +#include "absl/numeric/bits.h" +#include "absl/strings/string_view.h" +#include "rtc_base/bitstream_reader.h" +#include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { +namespace { +const size_t kVp9NumRefsPerFrame = 3; +const size_t kVp9MaxRefLFDeltas = 4; +const size_t kVp9MaxModeLFDeltas = 2; +const size_t kVp9MinTileWidthB64 = 4; +const size_t kVp9MaxTileWidthB64 = 64; + +void Vp9ReadColorConfig(BitstreamReader& br, + Vp9UncompressedHeader* frame_info) { + if (frame_info->profile == 2 || frame_info->profile == 3) { + frame_info->bit_detph = + br.Read<bool>() ? Vp9BitDept::k12Bit : Vp9BitDept::k10Bit; + } else { + frame_info->bit_detph = Vp9BitDept::k8Bit; + } + + frame_info->color_space = static_cast<Vp9ColorSpace>(br.ReadBits(3)); + + if (frame_info->color_space != Vp9ColorSpace::CS_RGB) { + frame_info->color_range = + br.Read<bool>() ? Vp9ColorRange::kFull : Vp9ColorRange::kStudio; + + if (frame_info->profile == 1 || frame_info->profile == 3) { + static constexpr Vp9YuvSubsampling kSubSamplings[] = { + Vp9YuvSubsampling::k444, Vp9YuvSubsampling::k440, + Vp9YuvSubsampling::k422, Vp9YuvSubsampling::k420}; + frame_info->sub_sampling = kSubSamplings[br.ReadBits(2)]; + + if (br.Read<bool>()) { + RTC_LOG(LS_WARNING) << "Failed to parse header. Reserved bit set."; + br.Invalidate(); + return; + } + } else { + // Profile 0 or 2. + frame_info->sub_sampling = Vp9YuvSubsampling::k420; + } + } else { + // SRGB + frame_info->color_range = Vp9ColorRange::kFull; + if (frame_info->profile == 1 || frame_info->profile == 3) { + frame_info->sub_sampling = Vp9YuvSubsampling::k444; + if (br.Read<bool>()) { + RTC_LOG(LS_WARNING) << "Failed to parse header. Reserved bit set."; + br.Invalidate(); + } + } else { + RTC_LOG(LS_WARNING) << "Failed to parse header. 4:4:4 color not supported" + " in profile 0 or 2."; + br.Invalidate(); + } + } +} + +void ReadRefreshFrameFlags(BitstreamReader& br, + Vp9UncompressedHeader* frame_info) { + // Refresh frame flags. + uint8_t flags = br.Read<uint8_t>(); + for (int i = 0; i < 8; ++i) { + frame_info->updated_buffers.set(i, (flags & (0x01 << (7 - i))) != 0); + } +} + +void Vp9ReadFrameSize(BitstreamReader& br, Vp9UncompressedHeader* frame_info) { + // 16 bits: frame (width|height) - 1. + frame_info->frame_width = br.Read<uint16_t>() + 1; + frame_info->frame_height = br.Read<uint16_t>() + 1; +} + +void Vp9ReadRenderSize(size_t total_buffer_size_bits, + BitstreamReader& br, + Vp9UncompressedHeader* frame_info) { + // render_and_frame_size_different + if (br.Read<bool>()) { + frame_info->render_size_offset_bits = + total_buffer_size_bits - br.RemainingBitCount(); + // 16 bits: render (width|height) - 1. + frame_info->render_width = br.Read<uint16_t>() + 1; + frame_info->render_height = br.Read<uint16_t>() + 1; + } else { + frame_info->render_height = frame_info->frame_height; + frame_info->render_width = frame_info->frame_width; + } +} + +void Vp9ReadFrameSizeFromRefs(BitstreamReader& br, + Vp9UncompressedHeader* frame_info) { + for (size_t i = 0; i < kVp9NumRefsPerFrame; i++) { + // Size in refs. + if (br.Read<bool>()) { + frame_info->infer_size_from_reference = frame_info->reference_buffers[i]; + return; + } + } + + Vp9ReadFrameSize(br, frame_info); +} + +void Vp9ReadLoopfilter(BitstreamReader& br) { + // 6 bits: filter level. + // 3 bits: sharpness level. + br.ConsumeBits(9); + + if (!br.Read<bool>()) { // mode_ref_delta_enabled + return; + } + if (!br.Read<bool>()) { // mode_ref_delta_update + return; + } + + for (size_t i = 0; i < kVp9MaxRefLFDeltas; i++) { + if (br.Read<bool>()) { // update_ref_delta + br.ConsumeBits(7); + } + } + for (size_t i = 0; i < kVp9MaxModeLFDeltas; i++) { + if (br.Read<bool>()) { // update_mode_delta + br.ConsumeBits(7); + } + } +} + +void Vp9ReadQp(BitstreamReader& br, Vp9UncompressedHeader* frame_info) { + frame_info->base_qp = br.Read<uint8_t>(); + + // yuv offsets + frame_info->is_lossless = frame_info->base_qp == 0; + for (int i = 0; i < 3; ++i) { + if (br.Read<bool>()) { // if delta_coded + // delta_q is a signed integer with leading 4 bits containing absolute + // value and last bit containing sign. There are are two ways to represent + // zero with such encoding. + if ((br.ReadBits(5) & 0b1111'0) != 0) { // delta_q + frame_info->is_lossless = false; + } + } + } +} + +void Vp9ReadSegmentationParams(BitstreamReader& br, + Vp9UncompressedHeader* frame_info) { + constexpr int kSegmentationFeatureBits[kVp9SegLvlMax] = {8, 6, 2, 0}; + constexpr bool kSegmentationFeatureSigned[kVp9SegLvlMax] = {true, true, false, + false}; + + frame_info->segmentation_enabled = br.Read<bool>(); + if (!frame_info->segmentation_enabled) { + return; + } + + if (br.Read<bool>()) { // update_map + frame_info->segmentation_tree_probs.emplace(); + for (int i = 0; i < 7; ++i) { + if (br.Read<bool>()) { + (*frame_info->segmentation_tree_probs)[i] = br.Read<uint8_t>(); + } else { + (*frame_info->segmentation_tree_probs)[i] = 255; + } + } + + // temporal_update + frame_info->segmentation_pred_prob.emplace(); + if (br.Read<bool>()) { + for (int i = 0; i < 3; ++i) { + if (br.Read<bool>()) { + (*frame_info->segmentation_pred_prob)[i] = br.Read<uint8_t>(); + } else { + (*frame_info->segmentation_pred_prob)[i] = 255; + } + } + } else { + frame_info->segmentation_pred_prob->fill(255); + } + } + + if (br.Read<bool>()) { // segmentation_update_data + frame_info->segmentation_is_delta = br.Read<bool>(); + for (size_t i = 0; i < kVp9MaxSegments; ++i) { + for (size_t j = 0; j < kVp9SegLvlMax; ++j) { + if (!br.Read<bool>()) { // feature_enabled + continue; + } + if (kSegmentationFeatureBits[j] == 0) { + // No feature bits used and no sign, just mark it and return. + frame_info->segmentation_features[i][j] = 1; + continue; + } + frame_info->segmentation_features[i][j] = + br.ReadBits(kSegmentationFeatureBits[j]); + if (kSegmentationFeatureSigned[j] && br.Read<bool>()) { + (*frame_info->segmentation_features[i][j]) *= -1; + } + } + } + } +} + +void Vp9ReadTileInfo(BitstreamReader& br, Vp9UncompressedHeader* frame_info) { + size_t mi_cols = (frame_info->frame_width + 7) >> 3; + size_t sb64_cols = (mi_cols + 7) >> 3; + + size_t min_log2 = 0; + while ((kVp9MaxTileWidthB64 << min_log2) < sb64_cols) { + ++min_log2; + } + + size_t max_log2 = 1; + while ((sb64_cols >> max_log2) >= kVp9MinTileWidthB64) { + ++max_log2; + } + --max_log2; + + frame_info->tile_cols_log2 = min_log2; + while (frame_info->tile_cols_log2 < max_log2) { + if (br.Read<bool>()) { + ++frame_info->tile_cols_log2; + } else { + break; + } + } + frame_info->tile_rows_log2 = 0; + if (br.Read<bool>()) { + ++frame_info->tile_rows_log2; + if (br.Read<bool>()) { + ++frame_info->tile_rows_log2; + } + } +} + +const Vp9InterpolationFilter kLiteralToType[4] = { + Vp9InterpolationFilter::kEightTapSmooth, Vp9InterpolationFilter::kEightTap, + Vp9InterpolationFilter::kEightTapSharp, Vp9InterpolationFilter::kBilinear}; +} // namespace + +std::string Vp9UncompressedHeader::ToString() const { + char buf[1024]; + rtc::SimpleStringBuilder oss(buf); + + oss << "Vp9UncompressedHeader { " + << "profile = " << profile; + + if (show_existing_frame) { + oss << ", show_existing_frame = " << *show_existing_frame << " }"; + return oss.str(); + } + + oss << ", frame type = " << (is_keyframe ? "key" : "delta") + << ", show_frame = " << (show_frame ? "true" : "false") + << ", error_resilient = " << (error_resilient ? "true" : "false"); + + oss << ", bit_depth = "; + switch (bit_detph) { + case Vp9BitDept::k8Bit: + oss << "8bit"; + break; + case Vp9BitDept::k10Bit: + oss << "10bit"; + break; + case Vp9BitDept::k12Bit: + oss << "12bit"; + break; + } + + if (color_space) { + oss << ", color_space = "; + switch (*color_space) { + case Vp9ColorSpace::CS_UNKNOWN: + oss << "unknown"; + break; + case Vp9ColorSpace::CS_BT_601: + oss << "CS_BT_601 Rec. ITU-R BT.601-7"; + break; + case Vp9ColorSpace::CS_BT_709: + oss << "Rec. ITU-R BT.709-6"; + break; + case Vp9ColorSpace::CS_SMPTE_170: + oss << "SMPTE-170"; + break; + case Vp9ColorSpace::CS_SMPTE_240: + oss << "SMPTE-240"; + break; + case Vp9ColorSpace::CS_BT_2020: + oss << "Rec. ITU-R BT.2020-2"; + break; + case Vp9ColorSpace::CS_RESERVED: + oss << "Reserved"; + break; + case Vp9ColorSpace::CS_RGB: + oss << "sRGB (IEC 61966-2-1)"; + break; + } + } + + if (color_range) { + oss << ", color_range = "; + switch (*color_range) { + case Vp9ColorRange::kFull: + oss << "full"; + break; + case Vp9ColorRange::kStudio: + oss << "studio"; + break; + } + } + + if (sub_sampling) { + oss << ", sub_sampling = "; + switch (*sub_sampling) { + case Vp9YuvSubsampling::k444: + oss << "444"; + break; + case Vp9YuvSubsampling::k440: + oss << "440"; + break; + case Vp9YuvSubsampling::k422: + oss << "422"; + break; + case Vp9YuvSubsampling::k420: + oss << "420"; + break; + } + } + + if (infer_size_from_reference) { + oss << ", infer_frame_resolution_from = " << *infer_size_from_reference; + } else { + oss << ", frame_width = " << frame_width + << ", frame_height = " << frame_height; + } + if (render_width != 0 && render_height != 0) { + oss << ", render_width = " << render_width + << ", render_height = " << render_height; + } + + oss << ", base qp = " << base_qp; + if (reference_buffers[0] != -1) { + oss << ", last_buffer = " << reference_buffers[0]; + } + if (reference_buffers[1] != -1) { + oss << ", golden_buffer = " << reference_buffers[1]; + } + if (reference_buffers[2] != -1) { + oss << ", altref_buffer = " << reference_buffers[2]; + } + + oss << ", updated buffers = { "; + bool first = true; + for (int i = 0; i < 8; ++i) { + if (updated_buffers.test(i)) { + if (first) { + first = false; + } else { + oss << ", "; + } + oss << i; + } + } + oss << " }"; + + oss << ", compressed_header_size_bytes = " << compressed_header_size; + + oss << " }"; + return oss.str(); +} + +void Parse(BitstreamReader& br, + Vp9UncompressedHeader* frame_info, + bool qp_only) { + const size_t total_buffer_size_bits = br.RemainingBitCount(); + + // Frame marker. + if (br.ReadBits(2) != 0b10) { + RTC_LOG(LS_WARNING) << "Failed to parse header. Frame marker should be 2."; + br.Invalidate(); + return; + } + + // Profile has low bit first. + frame_info->profile = br.ReadBit(); + frame_info->profile |= br.ReadBit() << 1; + if (frame_info->profile > 2 && br.Read<bool>()) { + RTC_LOG(LS_WARNING) + << "Failed to parse header. Unsupported bitstream profile."; + br.Invalidate(); + return; + } + + // Show existing frame. + if (br.Read<bool>()) { + frame_info->show_existing_frame = br.ReadBits(3); + return; + } + + // Frame type: KEY_FRAME(0), INTER_FRAME(1). + frame_info->is_keyframe = !br.Read<bool>(); + frame_info->show_frame = br.Read<bool>(); + frame_info->error_resilient = br.Read<bool>(); + + if (frame_info->is_keyframe) { + if (br.ReadBits(24) != 0x498342) { + RTC_LOG(LS_WARNING) << "Failed to parse header. Invalid sync code."; + br.Invalidate(); + return; + } + + Vp9ReadColorConfig(br, frame_info); + Vp9ReadFrameSize(br, frame_info); + Vp9ReadRenderSize(total_buffer_size_bits, br, frame_info); + + // Key-frames implicitly update all buffers. + frame_info->updated_buffers.set(); + } else { + // Non-keyframe. + bool is_intra_only = false; + if (!frame_info->show_frame) { + is_intra_only = br.Read<bool>(); + } + if (!frame_info->error_resilient) { + br.ConsumeBits(2); // Reset frame context. + } + + if (is_intra_only) { + if (br.ReadBits(24) != 0x498342) { + RTC_LOG(LS_WARNING) << "Failed to parse header. Invalid sync code."; + br.Invalidate(); + return; + } + + if (frame_info->profile > 0) { + Vp9ReadColorConfig(br, frame_info); + } else { + frame_info->color_space = Vp9ColorSpace::CS_BT_601; + frame_info->sub_sampling = Vp9YuvSubsampling::k420; + frame_info->bit_detph = Vp9BitDept::k8Bit; + } + frame_info->reference_buffers.fill(-1); + ReadRefreshFrameFlags(br, frame_info); + Vp9ReadFrameSize(br, frame_info); + Vp9ReadRenderSize(total_buffer_size_bits, br, frame_info); + } else { + ReadRefreshFrameFlags(br, frame_info); + + frame_info->reference_buffers_sign_bias[0] = false; + for (size_t i = 0; i < kVp9NumRefsPerFrame; i++) { + frame_info->reference_buffers[i] = br.ReadBits(3); + frame_info->reference_buffers_sign_bias[Vp9ReferenceFrame::kLast + i] = + br.Read<bool>(); + } + + Vp9ReadFrameSizeFromRefs(br, frame_info); + Vp9ReadRenderSize(total_buffer_size_bits, br, frame_info); + + frame_info->allow_high_precision_mv = br.Read<bool>(); + + // Interpolation filter. + if (br.Read<bool>()) { + frame_info->interpolation_filter = Vp9InterpolationFilter::kSwitchable; + } else { + frame_info->interpolation_filter = kLiteralToType[br.ReadBits(2)]; + } + } + } + + if (!frame_info->error_resilient) { + // 1 bit: Refresh frame context. + // 1 bit: Frame parallel decoding mode. + br.ConsumeBits(2); + } + + // Frame context index. + frame_info->frame_context_idx = br.ReadBits(2); + + Vp9ReadLoopfilter(br); + + // Read base QP. + Vp9ReadQp(br, frame_info); + + if (qp_only) { + // Not interested in the rest of the header, return early. + return; + } + + Vp9ReadSegmentationParams(br, frame_info); + Vp9ReadTileInfo(br, frame_info); + frame_info->compressed_header_size = br.Read<uint16_t>(); + frame_info->uncompressed_header_size = + (total_buffer_size_bits / 8) - (br.RemainingBitCount() / 8); +} + +absl::optional<Vp9UncompressedHeader> ParseUncompressedVp9Header( + rtc::ArrayView<const uint8_t> buf) { + BitstreamReader reader(buf); + Vp9UncompressedHeader frame_info; + Parse(reader, &frame_info, /*qp_only=*/false); + if (reader.Ok() && frame_info.frame_width > 0) { + return frame_info; + } + return absl::nullopt; +} + +namespace vp9 { + +bool GetQp(const uint8_t* buf, size_t length, int* qp) { + BitstreamReader reader(rtc::MakeArrayView(buf, length)); + Vp9UncompressedHeader frame_info; + Parse(reader, &frame_info, /*qp_only=*/true); + if (!reader.Ok()) { + return false; + } + *qp = frame_info.base_qp; + return true; +} + +} // namespace vp9 +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.h b/third_party/libwebrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.h new file mode 100644 index 0000000000..8d1b88c3d3 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.h @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_VP9_UNCOMPRESSED_HEADER_PARSER_H_ +#define MODULES_VIDEO_CODING_UTILITY_VP9_UNCOMPRESSED_HEADER_PARSER_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <array> +#include <bitset> +#include <string> + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "modules/video_coding/utility/vp9_constants.h" + +namespace webrtc { + +namespace vp9 { + +// Gets the QP, QP range: [0, 255]. +// Returns true on success, false otherwise. +bool GetQp(const uint8_t* buf, size_t length, int* qp); + +} // namespace vp9 + +// Bit depth per channel. Support varies by profile. +enum class Vp9BitDept : uint8_t { + k8Bit = 8, + k10Bit = 10, + k12Bit = 12, +}; + +enum class Vp9ColorSpace : uint8_t { + CS_UNKNOWN = 0, // Unknown (in this case the color space must be signaled + // outside the VP9 bitstream). + CS_BT_601 = 1, // CS_BT_601 Rec. ITU-R BT.601-7 + CS_BT_709 = 2, // Rec. ITU-R BT.709-6 + CS_SMPTE_170 = 3, // SMPTE-170 + CS_SMPTE_240 = 4, // SMPTE-240 + CS_BT_2020 = 5, // Rec. ITU-R BT.2020-2 + CS_RESERVED = 6, // Reserved + CS_RGB = 7, // sRGB (IEC 61966-2-1) +}; + +enum class Vp9ColorRange { + kStudio, // Studio swing: + // For BitDepth equals 8: + // Y is between 16 and 235 inclusive. + // U and V are between 16 and 240 inclusive. + // For BitDepth equals 10: + // Y is between 64 and 940 inclusive. + // U and V are between 64 and 960 inclusive. + // For BitDepth equals 12: + // Y is between 256 and 3760. + // U and V are between 256 and 3840 inclusive. + kFull // Full swing; no restriction on Y, U, V values. +}; + +enum class Vp9YuvSubsampling { + k444, + k440, + k422, + k420, +}; + +enum Vp9ReferenceFrame : int { + kNone = -1, + kIntra = 0, + kLast = 1, + kGolden = 2, + kAltref = 3, +}; + +enum class Vp9InterpolationFilter : uint8_t { + kEightTap = 0, + kEightTapSmooth = 1, + kEightTapSharp = 2, + kBilinear = 3, + kSwitchable = 4 +}; + +struct Vp9UncompressedHeader { + int profile = 0; // Profiles 0-3 are valid. + absl::optional<uint8_t> show_existing_frame; + bool is_keyframe = false; + bool show_frame = false; + bool error_resilient = false; + Vp9BitDept bit_detph = Vp9BitDept::k8Bit; + absl::optional<Vp9ColorSpace> color_space; + absl::optional<Vp9ColorRange> color_range; + absl::optional<Vp9YuvSubsampling> sub_sampling; + int frame_width = 0; + int frame_height = 0; + int render_width = 0; + int render_height = 0; + // Width/height of the tiles used (in units of 8x8 blocks). + size_t tile_cols_log2 = 0; // tile_cols = 1 << tile_cols_log2 + size_t tile_rows_log2 = 0; // tile_rows = 1 << tile_rows_log2 + absl::optional<size_t> render_size_offset_bits; + Vp9InterpolationFilter interpolation_filter = + Vp9InterpolationFilter::kEightTap; + bool allow_high_precision_mv = false; + int base_qp = 0; + bool is_lossless = false; + uint8_t frame_context_idx = 0; + + bool segmentation_enabled = false; + absl::optional<std::array<uint8_t, 7>> segmentation_tree_probs; + absl::optional<std::array<uint8_t, 3>> segmentation_pred_prob; + bool segmentation_is_delta = false; + std::array<std::array<absl::optional<int>, kVp9SegLvlMax>, kVp9MaxSegments> + segmentation_features; + + // Which of the 8 reference buffers may be used as references for this frame. + // -1 indicates not used (e.g. {-1, -1, -1} for intra-only frames). + std::array<int, kVp9RefsPerFrame> reference_buffers = {-1, -1, -1}; + // Sign bias corresponding to reference buffers, where the index is a + // ReferenceFrame. + // false/0 indidate backwards reference, true/1 indicate forwards reference). + std::bitset<kVp9MaxRefFrames> reference_buffers_sign_bias = 0; + + // Indicates which reference buffer [0,7] to infer the frame size from. + absl::optional<int> infer_size_from_reference; + // Which of the 8 reference buffers are updated by this frame. + std::bitset<kVp9NumRefFrames> updated_buffers = 0; + + // Header sizes, in bytes. + uint32_t uncompressed_header_size = 0; + uint32_t compressed_header_size = 0; + + bool is_intra_only() const { + return reference_buffers[0] == -1 && reference_buffers[1] == -1 && + reference_buffers[2] == -1; + } + + std::string ToString() const; +}; + +// Parses the uncompressed header and populates (most) values in a +// UncompressedHeader struct. Returns nullopt on failure. +absl::optional<Vp9UncompressedHeader> ParseUncompressedVp9Header( + rtc::ArrayView<const uint8_t> buf); + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_UTILITY_VP9_UNCOMPRESSED_HEADER_PARSER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/utility/vp9_uncompressed_header_parser_unittest.cc b/third_party/libwebrtc/modules/video_coding/utility/vp9_uncompressed_header_parser_unittest.cc new file mode 100644 index 0000000000..d8cc738e07 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/vp9_uncompressed_header_parser_unittest.cc @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" + +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace vp9 { +using ::testing::AllOf; +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::Field; +using ::testing::Optional; + +TEST(Vp9UncompressedHeaderParserTest, FrameWithSegmentation) { + // Uncompressed header from a frame generated with libvpx. + // Encoded QVGA frame (SL0 of a VGA frame) that includes a segmentation. + const uint8_t kHeader[] = { + 0x87, 0x01, 0x00, 0x00, 0x02, 0x7e, 0x01, 0xdf, 0x02, 0x7f, 0x01, 0xdf, + 0xc6, 0x87, 0x04, 0x83, 0x83, 0x2e, 0x46, 0x60, 0x20, 0x38, 0x0c, 0x06, + 0x03, 0xcd, 0x80, 0xc0, 0x60, 0x9f, 0xc5, 0x46, 0x00, 0x00, 0x00, 0x00, + 0x2e, 0x73, 0xb7, 0xee, 0x22, 0x06, 0x81, 0x82, 0xd4, 0xef, 0xc3, 0x58, + 0x1f, 0x12, 0xd2, 0x7b, 0x28, 0x1f, 0x80, 0xfc, 0x07, 0xe0, 0x00, 0x00}; + + absl::optional<Vp9UncompressedHeader> frame_info = + ParseUncompressedVp9Header(kHeader); + ASSERT_TRUE(frame_info.has_value()); + + EXPECT_FALSE(frame_info->is_keyframe); + EXPECT_TRUE(frame_info->error_resilient); + EXPECT_TRUE(frame_info->show_frame); + EXPECT_FALSE(frame_info->show_existing_frame); + EXPECT_EQ(frame_info->base_qp, 185); + EXPECT_EQ(frame_info->frame_width, 320); + EXPECT_EQ(frame_info->frame_height, 240); + EXPECT_EQ(frame_info->render_width, 640); + EXPECT_EQ(frame_info->render_height, 480); + EXPECT_TRUE(frame_info->allow_high_precision_mv); + EXPECT_EQ(frame_info->frame_context_idx, 0u); + EXPECT_EQ(frame_info->interpolation_filter, + Vp9InterpolationFilter::kSwitchable); + EXPECT_EQ(frame_info->is_lossless, false); + EXPECT_EQ(frame_info->profile, 0); + EXPECT_THAT(frame_info->reference_buffers, ElementsAre(0, 0, 0)); + EXPECT_THAT(frame_info->reference_buffers_sign_bias, 0b0000); + EXPECT_EQ(frame_info->updated_buffers, 0b10000000); + EXPECT_EQ(frame_info->tile_cols_log2, 0u); + EXPECT_EQ(frame_info->tile_rows_log2, 0u); + EXPECT_EQ(frame_info->render_size_offset_bits, 64u); + EXPECT_EQ(frame_info->compressed_header_size, 23u); + EXPECT_EQ(frame_info->uncompressed_header_size, 37u); + + EXPECT_TRUE(frame_info->segmentation_enabled); + EXPECT_FALSE(frame_info->segmentation_is_delta); + EXPECT_THAT(frame_info->segmentation_pred_prob, + Optional(ElementsAre(205, 1, 1))); + EXPECT_THAT(frame_info->segmentation_tree_probs, + Optional(ElementsAre(255, 255, 128, 1, 128, 128, 128))); + EXPECT_THAT(frame_info->segmentation_features[1][kVp9SegLvlAlt_Q], Eq(-63)); + EXPECT_THAT(frame_info->segmentation_features[2][kVp9SegLvlAlt_Q], Eq(-81)); +} + +TEST(Vp9UncompressedHeaderParserTest, SegmentationWithDefaultPredProbs) { + const uint8_t kHeader[] = {0x90, 0x49, 0x83, 0x42, 0x80, 0x2e, + 0x30, 0x0, 0xb0, 0x0, 0x37, 0xff, + 0x06, 0x80, 0x0, 0x0, 0x0, 0x0}; + absl::optional<Vp9UncompressedHeader> frame_info = + ParseUncompressedVp9Header(kHeader); + ASSERT_TRUE(frame_info.has_value()); + EXPECT_THAT(frame_info->segmentation_pred_prob, + Optional(ElementsAre(255, 255, 255))); +} + +TEST(Vp9UncompressedHeaderParserTest, SegmentationWithSkipLevel) { + const uint8_t kHeader[] = {0x90, 0x49, 0x83, 0x42, 0x80, 0x2e, 0x30, 0x00, + 0xb0, 0x00, 0x37, 0xff, 0x06, 0x80, 0x01, 0x08, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + absl::optional<Vp9UncompressedHeader> frame_info = + ParseUncompressedVp9Header(kHeader); + ASSERT_TRUE(frame_info.has_value()); + EXPECT_THAT(frame_info->segmentation_features[0][kVp9SegLvlSkip], Eq(1)); +} + +} // namespace vp9 +} // namespace webrtc |