611 lines
23 KiB
C++
611 lines
23 KiB
C++
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
|
|
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
* You can obtain one at http://mozilla.org/MPL/2.0/. */
|
|
|
|
#ifndef VideoFrameConverter_h
|
|
#define VideoFrameConverter_h
|
|
|
|
#include "ImageContainer.h"
|
|
#include "ImageConversion.h"
|
|
#include "Pacer.h"
|
|
#include "PerformanceRecorder.h"
|
|
#include "VideoSegment.h"
|
|
#include "nsISupportsImpl.h"
|
|
#include "nsThreadUtils.h"
|
|
#include "jsapi/RTCStatsReport.h"
|
|
#include "mozilla/dom/ImageBitmapBinding.h"
|
|
#include "mozilla/dom/ImageUtils.h"
|
|
#include "api/video/video_frame.h"
|
|
#include "common_video/include/video_frame_buffer_pool.h"
|
|
#include "common_video/include/video_frame_buffer.h"
|
|
#include "media/base/adapted_video_track_source.h"
|
|
|
|
// The number of frame buffers VideoFrameConverter may create before returning
|
|
// errors.
|
|
// Sometimes these are released synchronously but they can be forwarded all the
|
|
// way to the encoder for asynchronous encoding. With a pool size of 5,
|
|
// we allow 1 buffer for the current conversion, and 4 buffers to be queued at
|
|
// the encoder.
|
|
#define CONVERTER_BUFFER_POOL_SIZE 5
|
|
|
|
extern mozilla::LazyLogModule gMediaPipelineLog;
|
|
#define LOG(level, msg, ...) \
|
|
MOZ_LOG(gMediaPipelineLog, level, (msg, ##__VA_ARGS__))
|
|
|
|
namespace mozilla {
|
|
|
|
enum class FrameDroppingPolicy {
|
|
Allowed,
|
|
Disabled,
|
|
};
|
|
|
|
// An async video frame format converter.
|
|
//
|
|
// Input is typically a MediaTrackListener driven by MediaTrackGraph.
|
|
//
|
|
// Output is exposed through rtc::AdaptedVideoTrackSource, which implements
|
|
// rtc::VideoSourceInterface<webrtc::VideoFrame>.
|
|
template <FrameDroppingPolicy DropPolicy = FrameDroppingPolicy::Allowed>
|
|
class VideoFrameConverterImpl : public rtc::AdaptedVideoTrackSource {
|
|
protected:
|
|
explicit VideoFrameConverterImpl(
|
|
already_AddRefed<nsISerialEventTarget> aTarget,
|
|
const dom::RTCStatsTimestampMaker& aTimestampMaker, bool aLockScaling)
|
|
: mTimestampMaker(aTimestampMaker),
|
|
mTarget(aTarget),
|
|
mLockScaling(aLockScaling),
|
|
mPacer(MakeAndAddRef<Pacer<FrameToProcess>>(
|
|
do_AddRef(mTarget), mIdleFrameDuplicationInterval)),
|
|
mScalingPool(false, CONVERTER_BUFFER_POOL_SIZE),
|
|
mConversionPool(false, CONVERTER_BUFFER_POOL_SIZE) {
|
|
MOZ_COUNT_CTOR(VideoFrameConverterImpl);
|
|
}
|
|
|
|
// AdaptedVideoTrackSource impl -- we don't expect any of these to be called.
|
|
// They are in libwebrtc because they are used by blink to communicate
|
|
// properties from a video track source to their libwebrtc integration layer.
|
|
// We signal this elsewhere.
|
|
void GenerateKeyFrame() override {
|
|
MOZ_CRASH("Unexpected VideoFrameConverterImpl::GenerateKeyFrame");
|
|
}
|
|
SourceState state() const override {
|
|
MOZ_CRASH("Unexpected VideoFrameConverterImpl::state");
|
|
}
|
|
bool remote() const override {
|
|
MOZ_CRASH("Unexpected VideoFrameConverterImpl::remote");
|
|
}
|
|
bool is_screencast() const override {
|
|
MOZ_CRASH("Unexpected VideoFrameConverterImpl::is_screencast");
|
|
}
|
|
std::optional<bool> needs_denoising() const override {
|
|
MOZ_CRASH("Unexpected VideoFrameConverterImpl::needs_denoising");
|
|
}
|
|
|
|
void RegisterListener() {
|
|
mPacingListener = mPacer->PacedItemEvent().Connect(
|
|
mTarget,
|
|
[self = RefPtr(this)](FrameToProcess&& aFrame, TimeStamp aTime) {
|
|
self->QueueForProcessing(std::move(aFrame.mImage), aTime,
|
|
aFrame.mSize, aFrame.mForceBlack);
|
|
});
|
|
}
|
|
|
|
public:
|
|
using rtc::VideoSourceInterface<webrtc::VideoFrame>::AddOrUpdateSink;
|
|
using rtc::VideoSourceInterface<webrtc::VideoFrame>::RemoveSink;
|
|
|
|
void QueueVideoChunk(const VideoChunk& aChunk, bool aForceBlack) {
|
|
gfx::IntSize size = aChunk.mFrame.GetIntrinsicSize();
|
|
if (size.width == 0 || size.height == 0) {
|
|
return;
|
|
}
|
|
|
|
TimeStamp t = aChunk.mTimeStamp;
|
|
MOZ_ASSERT(!t.IsNull());
|
|
|
|
mPacer->Enqueue(
|
|
FrameToProcess(aChunk.mFrame.GetImage(), t, size, aForceBlack), t);
|
|
}
|
|
|
|
/**
|
|
* An active VideoFrameConverter actively converts queued video frames.
|
|
* While inactive, we keep track of the frame most recently queued for
|
|
* processing, so it can be immediately sent out once activated.
|
|
*/
|
|
void SetActive(bool aActive) {
|
|
MOZ_ALWAYS_SUCCEEDS(mTarget->Dispatch(NS_NewRunnableFunction(
|
|
__func__, [self = RefPtr<VideoFrameConverterImpl>(this), this, aActive,
|
|
time = TimeStamp::Now()] {
|
|
if (mActive == aActive) {
|
|
return;
|
|
}
|
|
LOG(LogLevel::Debug, "VideoFrameConverter %p is now %s", this,
|
|
aActive ? "active" : "inactive");
|
|
mActive = aActive;
|
|
if (aActive && mLastFrameQueuedForProcessing.Serial() != -2) {
|
|
// After activating, we re-process the last image that was queued
|
|
// for processing so it can be immediately sent. The image is reset
|
|
// so it doesn't get dropped if within the duplicate frame interval.
|
|
QueueForProcessing(std::move(mLastFrameQueuedForProcessing.mImage),
|
|
std::max(mLastFrameQueuedForProcessing.mTime +
|
|
TimeDuration::FromMicroseconds(1),
|
|
time),
|
|
mLastFrameQueuedForProcessing.mSize,
|
|
mLastFrameQueuedForProcessing.mForceBlack);
|
|
}
|
|
})));
|
|
}
|
|
|
|
void SetTrackEnabled(bool aTrackEnabled) {
|
|
MOZ_ALWAYS_SUCCEEDS(mTarget->Dispatch(NS_NewRunnableFunction(
|
|
__func__, [self = RefPtr<VideoFrameConverterImpl>(this), this,
|
|
aTrackEnabled, time = TimeStamp::Now()] {
|
|
if (mTrackEnabled == aTrackEnabled) {
|
|
return;
|
|
}
|
|
LOG(LogLevel::Debug, "VideoFrameConverterImpl %p Track is now %s",
|
|
this, aTrackEnabled ? "enabled" : "disabled");
|
|
mTrackEnabled = aTrackEnabled;
|
|
if (!aTrackEnabled) {
|
|
// After disabling we immediately send a frame as black, so it can
|
|
// be seen quickly, even if no frames are flowing. If no frame has
|
|
// been queued for processing yet, we use the FrameToProcess default
|
|
// size (640x480).
|
|
QueueForProcessing(/* aImage= */ nullptr,
|
|
std::max(mLastFrameQueuedForProcessing.mTime +
|
|
TimeDuration::FromMicroseconds(1),
|
|
time),
|
|
mLastFrameQueuedForProcessing.mSize,
|
|
/* aForceBlack= */ true);
|
|
}
|
|
})));
|
|
}
|
|
|
|
void SetTrackingId(TrackingId aTrackingId) {
|
|
MOZ_ALWAYS_SUCCEEDS(mTarget->Dispatch(NS_NewRunnableFunction(
|
|
__func__, [self = RefPtr<VideoFrameConverterImpl>(this), this,
|
|
id = std::move(aTrackingId)]() mutable {
|
|
mTrackingId = Some(std::move(id));
|
|
})));
|
|
}
|
|
|
|
void SetIdleFrameDuplicationInterval(TimeDuration aInterval) {
|
|
MOZ_ALWAYS_SUCCEEDS(mTarget->Dispatch(NS_NewRunnableFunction(
|
|
__func__, [self = RefPtr(this), this, aInterval] {
|
|
mIdleFrameDuplicationInterval = aInterval;
|
|
})));
|
|
mPacer->SetDuplicationInterval(aInterval);
|
|
}
|
|
|
|
void Shutdown() {
|
|
mPacer->Shutdown()->Then(
|
|
mTarget, __func__,
|
|
[self = RefPtr<VideoFrameConverterImpl>(this), this] {
|
|
mPacingListener.DisconnectIfExists();
|
|
mScalingPool.Release();
|
|
mConversionPool.Release();
|
|
mLastFrameQueuedForProcessing = FrameToProcess();
|
|
mLastFrameConverted = Nothing();
|
|
});
|
|
}
|
|
|
|
protected:
|
|
struct FrameToProcess {
|
|
FrameToProcess() = default;
|
|
|
|
FrameToProcess(RefPtr<layers::Image> aImage, TimeStamp aTime,
|
|
gfx::IntSize aSize, bool aForceBlack)
|
|
: mImage(std::move(aImage)),
|
|
mTime(aTime),
|
|
mSize(aSize),
|
|
mForceBlack(aForceBlack) {}
|
|
|
|
RefPtr<layers::Image> mImage;
|
|
TimeStamp mTime = TimeStamp::Now();
|
|
gfx::IntSize mSize = gfx::IntSize(640, 480);
|
|
bool mForceBlack = false;
|
|
|
|
int32_t Serial() const {
|
|
if (mForceBlack) {
|
|
// Set the last-img check to indicate black.
|
|
// -1 is not a guaranteed invalid serial. See bug 1262134.
|
|
return -1;
|
|
}
|
|
if (!mImage) {
|
|
// Set the last-img check to indicate reset.
|
|
// -2 is not a guaranteed invalid serial. See bug 1262134.
|
|
return -2;
|
|
}
|
|
return mImage->GetSerial();
|
|
}
|
|
};
|
|
|
|
struct FrameConverted {
|
|
FrameConverted(webrtc::VideoFrame aFrame, gfx::IntSize aOriginalSize,
|
|
int32_t aSerial)
|
|
: mFrame(std::move(aFrame)),
|
|
mOriginalSize(aOriginalSize),
|
|
mSerial(aSerial) {}
|
|
|
|
webrtc::VideoFrame mFrame;
|
|
gfx::IntSize mOriginalSize;
|
|
int32_t mSerial;
|
|
};
|
|
|
|
MOZ_COUNTED_DTOR_VIRTUAL(VideoFrameConverterImpl)
|
|
|
|
void VideoFrameConverted(const webrtc::VideoFrame& aVideoFrame,
|
|
gfx::IntSize aOriginalSize, int32_t aSerial) {
|
|
MOZ_ASSERT(mTarget->IsOnCurrentThread());
|
|
|
|
LOG(LogLevel::Verbose,
|
|
"VideoFrameConverterImpl %p: Converted a frame. Diff from last: %.3fms",
|
|
this,
|
|
static_cast<double>(aVideoFrame.timestamp_us() -
|
|
(mLastFrameConverted
|
|
? mLastFrameConverted->mFrame.timestamp_us()
|
|
: aVideoFrame.timestamp_us())) /
|
|
1000);
|
|
|
|
// Check that time doesn't go backwards
|
|
MOZ_ASSERT_IF(mLastFrameConverted,
|
|
aVideoFrame.timestamp_us() >
|
|
mLastFrameConverted->mFrame.timestamp_us());
|
|
|
|
mLastFrameConverted =
|
|
Some(FrameConverted(aVideoFrame, aOriginalSize, aSerial));
|
|
|
|
OnFrame(aVideoFrame);
|
|
}
|
|
|
|
void QueueForProcessing(RefPtr<layers::Image> aImage, TimeStamp aTime,
|
|
gfx::IntSize aSize, bool aForceBlack) {
|
|
MOZ_ASSERT(mTarget->IsOnCurrentThread());
|
|
|
|
FrameToProcess frame{std::move(aImage), aTime, aSize,
|
|
aForceBlack || !mTrackEnabled};
|
|
|
|
if (frame.mTime <= mLastFrameQueuedForProcessing.mTime) {
|
|
LOG(LogLevel::Debug,
|
|
"VideoFrameConverterImpl %p: Dropping a frame because time did not "
|
|
"progress (%.3fs)",
|
|
this,
|
|
(mLastFrameQueuedForProcessing.mTime - frame.mTime).ToSeconds());
|
|
return;
|
|
}
|
|
|
|
if (frame.Serial() == mLastFrameQueuedForProcessing.Serial()) {
|
|
// This is the same frame as the last one. We limit the same-frame rate,
|
|
// and rewrite the time so the frame-gap is in multiples of the
|
|
// duplication interval.
|
|
//
|
|
// The pacer only starts duplicating frames if there is no flow of frames
|
|
// into it. There are other reasons the same frame could repeat here, and
|
|
// at a shorter interval than the duplication interval. For instance after
|
|
// the sender is disabled (SetTrackEnabled) but there is still a flow of
|
|
// frames into the pacer. All disabled frames have the same serial.
|
|
if (auto diff = frame.mTime - mLastFrameQueuedForProcessing.mTime;
|
|
diff >= mIdleFrameDuplicationInterval) {
|
|
auto diff_us = static_cast<int64_t>(diff.ToMicroseconds());
|
|
auto idle_interval_us = static_cast<int64_t>(
|
|
mIdleFrameDuplicationInterval.ToMicroseconds());
|
|
auto multiples = diff_us / idle_interval_us;
|
|
MOZ_ASSERT(multiples > 0);
|
|
LOG(LogLevel::Verbose,
|
|
"VideoFrameConverterImpl %p: Rewrote time interval for a duplicate "
|
|
"frame from %.3fs to %.3fs",
|
|
this,
|
|
(frame.mTime - mLastFrameQueuedForProcessing.mTime).ToSeconds(),
|
|
(mIdleFrameDuplicationInterval * multiples).ToSeconds());
|
|
frame.mTime = mLastFrameQueuedForProcessing.mTime +
|
|
(mIdleFrameDuplicationInterval * multiples);
|
|
} else {
|
|
LOG(LogLevel::Verbose,
|
|
"VideoFrameConverterImpl %p: Dropping a duplicate frame because "
|
|
"the duplication interval (%.3fs) hasn't passed (%.3fs)",
|
|
this, mIdleFrameDuplicationInterval.ToSeconds(),
|
|
(frame.mTime - mLastFrameQueuedForProcessing.mTime).ToSeconds());
|
|
return;
|
|
}
|
|
}
|
|
|
|
mLastFrameQueuedForProcessing = std::move(frame);
|
|
|
|
if (!mActive) {
|
|
LOG(LogLevel::Debug,
|
|
"VideoFrameConverterImpl %p: Ignoring a frame because we're inactive",
|
|
this);
|
|
return;
|
|
}
|
|
|
|
MOZ_ALWAYS_SUCCEEDS(mTarget->Dispatch(NewRunnableMethod<FrameToProcess>(
|
|
"VideoFrameConverterImpl::ProcessVideoFrame", this,
|
|
&VideoFrameConverterImpl::ProcessVideoFrame,
|
|
mLastFrameQueuedForProcessing)));
|
|
}
|
|
|
|
void ProcessVideoFrame(const FrameToProcess& aFrame) {
|
|
MOZ_ASSERT(mTarget->IsOnCurrentThread());
|
|
|
|
auto convert = [this, &aFrame]() -> rtc::scoped_refptr<webrtc::I420Buffer> {
|
|
rtc::scoped_refptr<webrtc::I420Buffer> buffer =
|
|
mConversionPool.CreateI420Buffer(aFrame.mSize.width,
|
|
aFrame.mSize.height);
|
|
if (!buffer) {
|
|
#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
|
|
++mConversionFramesDropped;
|
|
#endif
|
|
MOZ_DIAGNOSTIC_ASSERT(mConversionFramesDropped <= 100,
|
|
"Conversion buffers must be leaking");
|
|
LOG(LogLevel::Warning,
|
|
"VideoFrameConverterImpl %p: Creating a conversion buffer failed",
|
|
this);
|
|
return nullptr;
|
|
}
|
|
|
|
#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
|
|
mConversionFramesDropped = 0;
|
|
#endif
|
|
PerformanceRecorder<CopyVideoStage> rec(
|
|
"VideoFrameConverterImpl::ConvertToI420"_ns, *mTrackingId,
|
|
buffer->width(), buffer->height());
|
|
nsresult rv = ConvertToI420(aFrame.mImage, buffer->MutableDataY(),
|
|
buffer->StrideY(), buffer->MutableDataU(),
|
|
buffer->StrideU(), buffer->MutableDataV(),
|
|
buffer->StrideV(), aFrame.mSize);
|
|
|
|
if (NS_FAILED(rv)) {
|
|
LOG(LogLevel::Warning,
|
|
"VideoFrameConverterImpl %p: Image conversion failed", this);
|
|
return nullptr;
|
|
}
|
|
rec.Record();
|
|
return buffer;
|
|
};
|
|
|
|
auto cropAndScale =
|
|
[this, &aFrame](
|
|
const rtc::scoped_refptr<webrtc::I420BufferInterface>& aSrc,
|
|
int aCrop_x, int aCrop_y, int aCrop_w, int aCrop_h, int aOut_width,
|
|
int aOut_height)
|
|
-> rtc::scoped_refptr<webrtc::I420BufferInterface> {
|
|
rtc::scoped_refptr<webrtc::I420Buffer> buffer =
|
|
mScalingPool.CreateI420Buffer(aOut_width, aOut_height);
|
|
if (!buffer) {
|
|
#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
|
|
++mScalingFramesDropped;
|
|
MOZ_DIAGNOSTIC_ASSERT(mScalingFramesDropped <= 100,
|
|
"Scaling buffers must be leaking");
|
|
#endif
|
|
LOG(LogLevel::Warning,
|
|
"VideoFrameConverterImpl %p: Creating a scaling buffer failed",
|
|
this);
|
|
return nullptr;
|
|
}
|
|
|
|
#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
|
|
mScalingFramesDropped = 0;
|
|
#endif
|
|
PerformanceRecorder<CopyVideoStage> rec(
|
|
"VideoFrameConverterImpl::CropAndScale"_ns, *mTrackingId,
|
|
aSrc->width(), aSrc->height());
|
|
LOG(LogLevel::Verbose,
|
|
"VideoFrameConverterImpl %p: Scaling image %d, %dx%d -> %dx%d", this,
|
|
aFrame.Serial(), aFrame.mSize.Width(), aFrame.mSize.Height(),
|
|
aOut_width, aOut_height);
|
|
buffer->CropAndScaleFrom(*aSrc, aCrop_x, aCrop_y, aCrop_w, aCrop_h);
|
|
rec.Record();
|
|
return buffer;
|
|
};
|
|
|
|
const webrtc::Timestamp time =
|
|
dom::RTCStatsTimestamp::FromMozTime(mTimestampMaker, aFrame.mTime)
|
|
.ToRealtime();
|
|
|
|
const bool sameAsLastConverted =
|
|
mLastFrameConverted && aFrame.Serial() == mLastFrameConverted->mSerial;
|
|
const gfx::IntSize inSize =
|
|
sameAsLastConverted ? mLastFrameConverted->mOriginalSize : aFrame.mSize;
|
|
|
|
int crop_x{}, crop_y{}, crop_width{}, crop_height{}, out_width{},
|
|
out_height{};
|
|
bool keep =
|
|
AdaptFrame(inSize.Width(), inSize.Height(), time.us(), &out_width,
|
|
&out_height, &crop_width, &crop_height, &crop_x, &crop_y);
|
|
|
|
if (mLockScaling) {
|
|
crop_x = crop_y = 0;
|
|
crop_width = out_width = inSize.Width();
|
|
crop_height = out_height = inSize.Height();
|
|
}
|
|
|
|
if (out_width == 0 || out_height == 0) {
|
|
LOG(LogLevel::Verbose,
|
|
"VideoFrameConverterImpl %p: Skipping a frame because it has no "
|
|
"pixels",
|
|
this);
|
|
OnFrameDropped();
|
|
return;
|
|
}
|
|
|
|
if constexpr (DropPolicy == FrameDroppingPolicy::Allowed) {
|
|
if (!keep) {
|
|
LOG(LogLevel::Verbose,
|
|
"VideoFrameConverterImpl %p: Dropping a frame because of SinkWants",
|
|
this);
|
|
// AdaptFrame has already called OnFrameDropped.
|
|
return;
|
|
}
|
|
if (aFrame.mTime < mLastFrameQueuedForProcessing.mTime) {
|
|
LOG(LogLevel::Verbose,
|
|
"VideoFrameConverterImpl %p: Dropping a frame that is %.3f seconds "
|
|
"before latest",
|
|
this,
|
|
(mLastFrameQueuedForProcessing.mTime - aFrame.mTime).ToSeconds());
|
|
OnFrameDropped();
|
|
return;
|
|
}
|
|
}
|
|
|
|
if (sameAsLastConverted) {
|
|
if (out_width == mLastFrameConverted->mFrame.width() &&
|
|
out_height == mLastFrameConverted->mFrame.height()) {
|
|
// This is the same input frame as last time. Avoid a conversion.
|
|
LOG(LogLevel::Verbose,
|
|
"VideoFrameConverterImpl %p: Re-converting last frame %d. "
|
|
"Re-using with same resolution.",
|
|
this, aFrame.Serial());
|
|
webrtc::VideoFrame frame = mLastFrameConverted->mFrame;
|
|
frame.set_timestamp_us(time.us());
|
|
VideoFrameConverted(frame, mLastFrameConverted->mOriginalSize,
|
|
mLastFrameConverted->mSerial);
|
|
return;
|
|
}
|
|
}
|
|
|
|
if (aFrame.mForceBlack) {
|
|
// Send a black image.
|
|
rtc::scoped_refptr<webrtc::I420Buffer> buffer =
|
|
mScalingPool.CreateI420Buffer(out_width, out_height);
|
|
if (!buffer) {
|
|
MOZ_DIAGNOSTIC_CRASH(
|
|
"Buffers not leaving scope except for "
|
|
"reconfig, should never leak");
|
|
LOG(LogLevel::Warning,
|
|
"VideoFrameConverterImpl %p: Creating a buffer for a black video "
|
|
"frame failed",
|
|
this);
|
|
OnFrameDropped();
|
|
return;
|
|
}
|
|
|
|
LOG(LogLevel::Verbose,
|
|
"VideoFrameConverterImpl %p: Sending a black video frame. "
|
|
"CropAndScale: %dx%d -> %dx%d",
|
|
this, aFrame.mSize.Width(), aFrame.mSize.Height(), out_width,
|
|
out_height);
|
|
webrtc::I420Buffer::SetBlack(buffer.get());
|
|
|
|
VideoFrameConverted(webrtc::VideoFrame::Builder()
|
|
.set_video_frame_buffer(buffer)
|
|
.set_timestamp_us(time.us())
|
|
.build(),
|
|
inSize, aFrame.Serial());
|
|
return;
|
|
}
|
|
|
|
if (!aFrame.mImage) {
|
|
// Don't send anything for null images.
|
|
return;
|
|
}
|
|
|
|
MOZ_ASSERT(aFrame.mImage->GetSize() == aFrame.mSize);
|
|
|
|
rtc::scoped_refptr<webrtc::I420BufferInterface> srcFrame;
|
|
RefPtr<layers::PlanarYCbCrImage> image =
|
|
aFrame.mImage->AsPlanarYCbCrImage();
|
|
if (image) {
|
|
dom::ImageUtils utils(image);
|
|
Maybe<dom::ImageBitmapFormat> format = utils.GetFormat();
|
|
if (format.isSome() &&
|
|
format.value() == dom::ImageBitmapFormat::YUV420P &&
|
|
image->GetData()) {
|
|
const layers::PlanarYCbCrData* data = image->GetData();
|
|
srcFrame = webrtc::WrapI420Buffer(
|
|
aFrame.mImage->GetSize().width, aFrame.mImage->GetSize().height,
|
|
data->mYChannel, data->mYStride, data->mCbChannel,
|
|
data->mCbCrStride, data->mCrChannel, data->mCbCrStride,
|
|
[image] { /* keep reference alive*/ });
|
|
|
|
LOG(LogLevel::Verbose,
|
|
"VideoFrameConverterImpl %p: Avoiding a conversion for image %d",
|
|
this, aFrame.Serial());
|
|
}
|
|
}
|
|
|
|
if (!srcFrame) {
|
|
srcFrame = convert();
|
|
}
|
|
|
|
if (!srcFrame) {
|
|
OnFrameDropped();
|
|
return;
|
|
}
|
|
|
|
if (srcFrame->width() == out_width && srcFrame->height() == out_height) {
|
|
LOG(LogLevel::Verbose,
|
|
"VideoFrameConverterImpl %p: Avoiding scaling for image %d, "
|
|
"Dimensions: %dx%d",
|
|
this, aFrame.Serial(), out_width, out_height);
|
|
VideoFrameConverted(webrtc::VideoFrame::Builder()
|
|
.set_video_frame_buffer(srcFrame)
|
|
.set_timestamp_us(time.us())
|
|
.build(),
|
|
inSize, aFrame.Serial());
|
|
return;
|
|
}
|
|
|
|
if (rtc::scoped_refptr<webrtc::I420BufferInterface> buffer =
|
|
cropAndScale(rtc::scoped_refptr(srcFrame), crop_x, crop_y,
|
|
crop_width, crop_height, out_width, out_height)) {
|
|
VideoFrameConverted(webrtc::VideoFrame::Builder()
|
|
.set_video_frame_buffer(buffer)
|
|
.set_timestamp_us(time.us())
|
|
.build(),
|
|
inSize, aFrame.Serial());
|
|
}
|
|
}
|
|
|
|
public:
|
|
const dom::RTCStatsTimestampMaker mTimestampMaker;
|
|
const nsCOMPtr<nsISerialEventTarget> mTarget;
|
|
const bool mLockScaling;
|
|
|
|
protected:
|
|
TimeDuration mIdleFrameDuplicationInterval = TimeDuration::Forever();
|
|
|
|
// Used to pace future frames close to their rendering-time. Thread-safe.
|
|
const RefPtr<Pacer<FrameToProcess>> mPacer;
|
|
|
|
// Accessed only from mTarget.
|
|
MediaEventListener mPacingListener;
|
|
webrtc::VideoFrameBufferPool mScalingPool;
|
|
webrtc::VideoFrameBufferPool mConversionPool;
|
|
FrameToProcess mLastFrameQueuedForProcessing;
|
|
Maybe<FrameConverted> mLastFrameConverted;
|
|
bool mActive = false;
|
|
bool mTrackEnabled = true;
|
|
Maybe<TrackingId> mTrackingId;
|
|
#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
|
|
size_t mConversionFramesDropped = 0;
|
|
size_t mScalingFramesDropped = 0;
|
|
#endif
|
|
};
|
|
|
|
class VideoFrameConverter
|
|
: public rtc::RefCountedObject<
|
|
VideoFrameConverterImpl<FrameDroppingPolicy::Allowed>> {
|
|
protected:
|
|
VideoFrameConverter(already_AddRefed<nsISerialEventTarget> aTarget,
|
|
const dom::RTCStatsTimestampMaker& aTimestampMaker,
|
|
bool aLockScaling)
|
|
: rtc::RefCountedObject<VideoFrameConverterImpl>(
|
|
std::move(aTarget), aTimestampMaker, aLockScaling) {}
|
|
|
|
public:
|
|
static already_AddRefed<VideoFrameConverter> Create(
|
|
already_AddRefed<nsISerialEventTarget> aTarget,
|
|
const dom::RTCStatsTimestampMaker& aTimestampMaker, bool aLockScaling) {
|
|
RefPtr<VideoFrameConverter> converter = new VideoFrameConverter(
|
|
std::move(aTarget), aTimestampMaker, aLockScaling);
|
|
converter->RegisterListener();
|
|
return converter.forget();
|
|
}
|
|
};
|
|
|
|
} // namespace mozilla
|
|
|
|
#undef LOG
|
|
|
|
#endif // VideoFrameConverter_h
|