diff options
Diffstat (limited to 'third_party/libwebrtc/modules/video_capture/linux/video_capture_pipewire.cc')
-rw-r--r-- | third_party/libwebrtc/modules/video_capture/linux/video_capture_pipewire.cc | 414 |
1 files changed, 414 insertions, 0 deletions
diff --git a/third_party/libwebrtc/modules/video_capture/linux/video_capture_pipewire.cc b/third_party/libwebrtc/modules/video_capture/linux/video_capture_pipewire.cc new file mode 100644 index 0000000000..8af483636a --- /dev/null +++ b/third_party/libwebrtc/modules/video_capture/linux/video_capture_pipewire.cc @@ -0,0 +1,414 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_capture/linux/video_capture_pipewire.h" + +#include <spa/param/format.h> +#include <spa/param/video/format-utils.h> +#include <spa/pod/builder.h> +#include <spa/utils/result.h> + +#include <vector> + +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "modules/portal/pipewire_utils.h" +#include "rtc_base/logging.h" +#include "rtc_base/string_to_number.h" + +namespace webrtc { +namespace videocapturemodule { + +struct { + uint32_t spa_format; + VideoType video_type; +} constexpr kSupportedFormats[] = { + {SPA_VIDEO_FORMAT_I420, VideoType::kI420}, + {SPA_VIDEO_FORMAT_NV12, VideoType::kNV12}, + {SPA_VIDEO_FORMAT_YUY2, VideoType::kYUY2}, + {SPA_VIDEO_FORMAT_UYVY, VideoType::kUYVY}, + {SPA_VIDEO_FORMAT_RGB, VideoType::kRGB24}, +}; + +VideoType VideoCaptureModulePipeWire::PipeWireRawFormatToVideoType( + uint32_t spa_format) { + for (const auto& spa_and_pixel_format : kSupportedFormats) { + if (spa_and_pixel_format.spa_format == spa_format) + return spa_and_pixel_format.video_type; + } + RTC_LOG(LS_INFO) << "Unsupported pixel format: " << spa_format; + return VideoType::kUnknown; +} + +VideoCaptureModulePipeWire::VideoCaptureModulePipeWire( + VideoCaptureOptions* options) + : VideoCaptureImpl(), + session_(options->pipewire_session()), + initialized_(false), + started_(false) {} + +VideoCaptureModulePipeWire::~VideoCaptureModulePipeWire() { + RTC_DCHECK_RUN_ON(&api_checker_); + + StopCapture(); +} + +int32_t VideoCaptureModulePipeWire::Init(const char* deviceUniqueId) { + RTC_CHECK_RUNS_SERIALIZED(&capture_checker_); + RTC_DCHECK_RUN_ON(&api_checker_); + + absl::optional<int> id; + id = rtc::StringToNumber<int>(deviceUniqueId); + if (id == absl::nullopt) + return -1; + + node_id_ = id.value(); + + const int len = strlen(deviceUniqueId); + _deviceUniqueId = new (std::nothrow) char[len + 1]; + memcpy(_deviceUniqueId, deviceUniqueId, len + 1); + + return 0; +} + +static spa_pod* BuildFormat(spa_pod_builder* builder, + uint32_t format, + uint32_t width, + uint32_t height, + float frame_rate) { + spa_pod_frame frames[2]; + + spa_pod_builder_push_object(builder, &frames[0], SPA_TYPE_OBJECT_Format, + SPA_PARAM_EnumFormat); + spa_pod_builder_add(builder, SPA_FORMAT_mediaType, + SPA_POD_Id(SPA_MEDIA_TYPE_video), SPA_FORMAT_mediaSubtype, + SPA_POD_Id(format), 0); + + if (format == SPA_MEDIA_SUBTYPE_raw) { + spa_pod_builder_prop(builder, SPA_FORMAT_VIDEO_format, 0); + spa_pod_builder_push_choice(builder, &frames[1], SPA_CHOICE_Enum, 0); + spa_pod_builder_id(builder, kSupportedFormats[0].spa_format); + for (const auto& spa_and_pixel_format : kSupportedFormats) + spa_pod_builder_id(builder, spa_and_pixel_format.spa_format); + spa_pod_builder_pop(builder, &frames[1]); + } + + spa_rectangle preferred_size = spa_rectangle{width, height}; + spa_rectangle min_size = spa_rectangle{1, 1}; + spa_rectangle max_size = spa_rectangle{4096, 4096}; + spa_pod_builder_add( + builder, SPA_FORMAT_VIDEO_size, + SPA_POD_CHOICE_RANGE_Rectangle(&preferred_size, &min_size, &max_size), 0); + + spa_fraction preferred_frame_rate = + spa_fraction{static_cast<uint32_t>(frame_rate), 1}; + spa_fraction min_frame_rate = spa_fraction{0, 1}; + spa_fraction max_frame_rate = spa_fraction{INT32_MAX, 1}; + spa_pod_builder_add( + builder, SPA_FORMAT_VIDEO_framerate, + SPA_POD_CHOICE_RANGE_Fraction(&preferred_frame_rate, &min_frame_rate, + &max_frame_rate), + 0); + + return static_cast<spa_pod*>(spa_pod_builder_pop(builder, &frames[0])); +} + +int32_t VideoCaptureModulePipeWire::StartCapture( + const VideoCaptureCapability& capability) { + RTC_DCHECK_RUN_ON(&api_checker_); + + if (initialized_) { + if (capability == _requestedCapability) { + return 0; + } else { + StopCapture(); + } + } + + uint8_t buffer[1024] = {}; + + // We don't want members above to be guarded by capture_checker_ as + // it's meant to be for members that are accessed on the API thread + // only when we are not capturing. The code above can be called many + // times while sharing instance of VideoCapturePipeWire between + // websites and therefore it would not follow the requirements of this + // checker. + RTC_CHECK_RUNS_SERIALIZED(&capture_checker_); + PipeWireThreadLoopLock thread_loop_lock(session_->pw_main_loop_); + + RTC_LOG(LS_VERBOSE) << "Creating new PipeWire stream for node " << node_id_; + + pw_properties* reuse_props = + pw_properties_new_string("pipewire.client.reuse=1"); + stream_ = pw_stream_new(session_->pw_core_, "camera-stream", reuse_props); + + if (!stream_) { + RTC_LOG(LS_ERROR) << "Failed to create camera stream!"; + return -1; + } + + static const pw_stream_events stream_events{ + .version = PW_VERSION_STREAM_EVENTS, + .state_changed = &OnStreamStateChanged, + .param_changed = &OnStreamParamChanged, + .process = &OnStreamProcess, + }; + + pw_stream_add_listener(stream_, &stream_listener_, &stream_events, this); + + spa_pod_builder builder = spa_pod_builder{buffer, sizeof(buffer)}; + std::vector<const spa_pod*> params; + uint32_t width = capability.width; + uint32_t height = capability.height; + uint32_t frame_rate = capability.maxFPS; + bool prefer_jpeg = (width > 640) || (height > 480); + + params.push_back( + BuildFormat(&builder, SPA_MEDIA_SUBTYPE_raw, width, height, frame_rate)); + params.insert( + prefer_jpeg ? params.begin() : params.end(), + BuildFormat(&builder, SPA_MEDIA_SUBTYPE_mjpg, width, height, frame_rate)); + + int res = pw_stream_connect( + stream_, PW_DIRECTION_INPUT, node_id_, + static_cast<enum pw_stream_flags>(PW_STREAM_FLAG_AUTOCONNECT | + PW_STREAM_FLAG_DONT_RECONNECT | + PW_STREAM_FLAG_MAP_BUFFERS), + params.data(), params.size()); + if (res != 0) { + RTC_LOG(LS_ERROR) << "Could not connect to camera stream: " + << spa_strerror(res); + return -1; + } + + _requestedCapability = capability; + initialized_ = true; + + return 0; +} + +int32_t VideoCaptureModulePipeWire::StopCapture() { + RTC_DCHECK_RUN_ON(&api_checker_); + + PipeWireThreadLoopLock thread_loop_lock(session_->pw_main_loop_); + // PipeWireSession is guarded by API checker so just make sure we do + // race detection when the PipeWire loop is locked/stopped to not run + // any callback at this point. + RTC_CHECK_RUNS_SERIALIZED(&capture_checker_); + if (stream_) { + pw_stream_destroy(stream_); + stream_ = nullptr; + } + + _requestedCapability = VideoCaptureCapability(); + return 0; +} + +bool VideoCaptureModulePipeWire::CaptureStarted() { + RTC_DCHECK_RUN_ON(&api_checker_); + MutexLock lock(&api_lock_); + + return started_; +} + +int32_t VideoCaptureModulePipeWire::CaptureSettings( + VideoCaptureCapability& settings) { + RTC_DCHECK_RUN_ON(&api_checker_); + + settings = _requestedCapability; + + return 0; +} + +void VideoCaptureModulePipeWire::OnStreamParamChanged( + void* data, + uint32_t id, + const struct spa_pod* format) { + VideoCaptureModulePipeWire* that = + static_cast<VideoCaptureModulePipeWire*>(data); + RTC_DCHECK(that); + RTC_CHECK_RUNS_SERIALIZED(&that->capture_checker_); + + if (format && id == SPA_PARAM_Format) + that->OnFormatChanged(format); +} + +void VideoCaptureModulePipeWire::OnFormatChanged(const struct spa_pod* format) { + RTC_CHECK_RUNS_SERIALIZED(&capture_checker_); + + uint32_t media_type, media_subtype; + + if (spa_format_parse(format, &media_type, &media_subtype) < 0) { + RTC_LOG(LS_ERROR) << "Failed to parse video format."; + return; + } + + switch (media_subtype) { + case SPA_MEDIA_SUBTYPE_raw: { + struct spa_video_info_raw f; + spa_format_video_raw_parse(format, &f); + configured_capability_.width = f.size.width; + configured_capability_.height = f.size.height; + configured_capability_.videoType = PipeWireRawFormatToVideoType(f.format); + configured_capability_.maxFPS = f.framerate.num / f.framerate.denom; + break; + } + case SPA_MEDIA_SUBTYPE_mjpg: { + struct spa_video_info_mjpg f; + spa_format_video_mjpg_parse(format, &f); + configured_capability_.width = f.size.width; + configured_capability_.height = f.size.height; + configured_capability_.videoType = VideoType::kMJPEG; + configured_capability_.maxFPS = f.framerate.num / f.framerate.denom; + break; + } + default: + configured_capability_.videoType = VideoType::kUnknown; + } + + if (configured_capability_.videoType == VideoType::kUnknown) { + RTC_LOG(LS_ERROR) << "Unsupported video format."; + return; + } + + RTC_LOG(LS_VERBOSE) << "Configured capture format = " + << static_cast<int>(configured_capability_.videoType); + + uint8_t buffer[1024] = {}; + auto builder = spa_pod_builder{buffer, sizeof(buffer)}; + + // Setup buffers and meta header for new format. + std::vector<const spa_pod*> params; + spa_pod_frame frame; + spa_pod_builder_push_object(&builder, &frame, SPA_TYPE_OBJECT_ParamBuffers, + SPA_PARAM_Buffers); + + if (media_subtype == SPA_MEDIA_SUBTYPE_raw) { + // Enforce stride without padding. + size_t stride; + switch (configured_capability_.videoType) { + case VideoType::kI420: + case VideoType::kNV12: + stride = configured_capability_.width; + break; + case VideoType::kYUY2: + case VideoType::kUYVY: + stride = configured_capability_.width * 2; + break; + case VideoType::kRGB24: + stride = configured_capability_.width * 3; + break; + default: + RTC_LOG(LS_ERROR) << "Unsupported video format."; + return; + } + spa_pod_builder_add(&builder, SPA_PARAM_BUFFERS_stride, SPA_POD_Int(stride), + 0); + } + + spa_pod_builder_add( + &builder, SPA_PARAM_BUFFERS_buffers, SPA_POD_CHOICE_RANGE_Int(8, 1, 32), + SPA_PARAM_BUFFERS_dataType, + SPA_POD_CHOICE_FLAGS_Int((1 << SPA_DATA_MemFd) | (1 << SPA_DATA_MemPtr)), + 0); + params.push_back( + static_cast<spa_pod*>(spa_pod_builder_pop(&builder, &frame))); + + params.push_back(reinterpret_cast<spa_pod*>(spa_pod_builder_add_object( + &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, + SPA_POD_Id(SPA_META_Header), SPA_PARAM_META_size, + SPA_POD_Int(sizeof(struct spa_meta_header))))); + params.push_back(reinterpret_cast<spa_pod*>(spa_pod_builder_add_object( + &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, + SPA_POD_Id(SPA_META_VideoTransform), SPA_PARAM_META_size, + SPA_POD_Int(sizeof(struct spa_meta_videotransform))))); + pw_stream_update_params(stream_, params.data(), params.size()); +} + +void VideoCaptureModulePipeWire::OnStreamStateChanged( + void* data, + pw_stream_state old_state, + pw_stream_state state, + const char* error_message) { + VideoCaptureModulePipeWire* that = + static_cast<VideoCaptureModulePipeWire*>(data); + RTC_DCHECK(that); + + MutexLock lock(&that->api_lock_); + switch (state) { + case PW_STREAM_STATE_STREAMING: + that->started_ = true; + break; + case PW_STREAM_STATE_ERROR: + RTC_LOG(LS_ERROR) << "PipeWire stream state error: " << error_message; + [[fallthrough]]; + case PW_STREAM_STATE_PAUSED: + case PW_STREAM_STATE_UNCONNECTED: + case PW_STREAM_STATE_CONNECTING: + that->started_ = false; + break; + } + RTC_LOG(LS_VERBOSE) << "PipeWire stream state change: " + << pw_stream_state_as_string(old_state) << " -> " + << pw_stream_state_as_string(state); +} + +void VideoCaptureModulePipeWire::OnStreamProcess(void* data) { + VideoCaptureModulePipeWire* that = + static_cast<VideoCaptureModulePipeWire*>(data); + RTC_DCHECK(that); + RTC_CHECK_RUNS_SERIALIZED(&that->capture_checker_); + that->ProcessBuffers(); +} + +static VideoRotation VideorotationFromPipeWireTransform(uint32_t transform) { + switch (transform) { + case SPA_META_TRANSFORMATION_90: + return kVideoRotation_90; + case SPA_META_TRANSFORMATION_180: + return kVideoRotation_180; + case SPA_META_TRANSFORMATION_270: + return kVideoRotation_270; + default: + return kVideoRotation_0; + } +} + +void VideoCaptureModulePipeWire::ProcessBuffers() { + RTC_CHECK_RUNS_SERIALIZED(&capture_checker_); + + while (pw_buffer* buffer = pw_stream_dequeue_buffer(stream_)) { + struct spa_meta_header* h; + h = static_cast<struct spa_meta_header*>( + spa_buffer_find_meta_data(buffer->buffer, SPA_META_Header, sizeof(*h))); + + struct spa_meta_videotransform* videotransform; + videotransform = + static_cast<struct spa_meta_videotransform*>(spa_buffer_find_meta_data( + buffer->buffer, SPA_META_VideoTransform, sizeof(*videotransform))); + if (videotransform) { + VideoRotation rotation = + VideorotationFromPipeWireTransform(videotransform->transform); + SetCaptureRotation(rotation); + SetApplyRotation(rotation != kVideoRotation_0); + } + + if (h->flags & SPA_META_HEADER_FLAG_CORRUPTED) { + RTC_LOG(LS_INFO) << "Dropping corruped frame."; + } else { + IncomingFrame(static_cast<unsigned char*>(buffer->buffer->datas[0].data), + buffer->buffer->datas[0].chunk->size, + configured_capability_); + } + pw_stream_queue_buffer(stream_, buffer); + } +} + +} // namespace videocapturemodule +} // namespace webrtc |