/* * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "modules/desktop_capture/desktop_and_cursor_composer.h" #include #include #include #include #include #include "modules/desktop_capture/desktop_capturer.h" #include "modules/desktop_capture/desktop_frame.h" #include "modules/desktop_capture/mouse_cursor.h" #include "modules/desktop_capture/shared_desktop_frame.h" #include "rtc_base/arraysize.h" #include "test/gmock.h" #include "test/gtest.h" namespace webrtc { namespace { using testing::ElementsAre; const int kFrameXCoord = 100; const int kFrameYCoord = 200; const int kScreenWidth = 100; const int kScreenHeight = 100; const int kCursorWidth = 10; const int kCursorHeight = 10; const int kTestCursorSize = 3; const uint32_t kTestCursorData[kTestCursorSize][kTestCursorSize] = { { 0xffffffff, 0x99990000, 0xaa222222, }, { 0x88008800, 0xaa0000aa, 0xaa333333, }, { 0x00000000, 0xaa0000aa, 0xaa333333, }, }; uint32_t GetFakeFramePixelValue(const DesktopVector& p) { uint32_t r = 100 + p.x(); uint32_t g = 100 + p.y(); uint32_t b = 100 + p.x() + p.y(); return b + (g << 8) + (r << 16) + 0xff000000; } uint32_t GetFramePixel(const DesktopFrame& frame, const DesktopVector& pos) { return *reinterpret_cast(frame.GetFrameDataAtPos(pos)); } // Blends two pixel values taking into account alpha. uint32_t BlendPixels(uint32_t dest, uint32_t src) { uint8_t alpha = 255 - ((src & 0xff000000) >> 24); uint32_t r = ((dest & 0x00ff0000) >> 16) * alpha / 255 + ((src & 0x00ff0000) >> 16); uint32_t g = ((dest & 0x0000ff00) >> 8) * alpha / 255 + ((src & 0x0000ff00) >> 8); uint32_t b = (dest & 0x000000ff) * alpha / 255 + (src & 0x000000ff); return b + (g << 8) + (r << 16) + 0xff000000; } DesktopFrame* CreateTestFrame(int width = kScreenWidth, int height = kScreenHeight) { DesktopFrame* frame = new BasicDesktopFrame(DesktopSize(width, height)); uint32_t* data = reinterpret_cast(frame->data()); for (int y = 0; y < height; ++y) { for (int x = 0; x < width; ++x) { *(data++) = GetFakeFramePixelValue(DesktopVector(x, y)); } } return frame; } MouseCursor* CreateTestCursor(DesktopVector hotspot) { std::unique_ptr image( new BasicDesktopFrame(DesktopSize(kCursorWidth, kCursorHeight))); uint32_t* data = reinterpret_cast(image->data()); // Set four pixels near the hotspot and leave all other blank. for (int y = 0; y < kTestCursorSize; ++y) { for (int x = 0; x < kTestCursorSize; ++x) { data[(hotspot.y() + y) * kCursorWidth + (hotspot.x() + x)] = kTestCursorData[y][x]; } } return new MouseCursor(image.release(), hotspot); } class FakeScreenCapturer : public DesktopCapturer { public: FakeScreenCapturer() {} void Start(Callback* callback) override { callback_ = callback; } void CaptureFrame() override { callback_->OnCaptureResult( next_frame_ ? Result::SUCCESS : Result::ERROR_TEMPORARY, std::move(next_frame_)); } void SetNextFrame(std::unique_ptr next_frame) { next_frame_ = std::move(next_frame); } bool IsOccluded(const DesktopVector& pos) override { return is_occluded_; } void set_is_occluded(bool value) { is_occluded_ = value; } private: Callback* callback_ = nullptr; std::unique_ptr next_frame_; bool is_occluded_ = false; }; class FakeMouseMonitor : public MouseCursorMonitor { public: FakeMouseMonitor() : changed_(true) {} void SetState(CursorState state, const DesktopVector& pos) { state_ = state; position_ = pos; } void SetHotspot(const DesktopVector& hotspot) { if (!hotspot_.equals(hotspot)) changed_ = true; hotspot_ = hotspot; } void Init(Callback* callback, Mode mode) override { callback_ = callback; } void Capture() override { if (changed_) { callback_->OnMouseCursor(CreateTestCursor(hotspot_)); } callback_->OnMouseCursorPosition(position_); } private: Callback* callback_; CursorState state_; DesktopVector position_; DesktopVector hotspot_; bool changed_; }; void VerifyFrame(const DesktopFrame& frame, MouseCursorMonitor::CursorState state, const DesktopVector& pos) { // Verify that all other pixels are set to their original values. DesktopRect image_rect = DesktopRect::MakeWH(kTestCursorSize, kTestCursorSize); image_rect.Translate(pos); for (int y = 0; y < kScreenHeight; ++y) { for (int x = 0; x < kScreenWidth; ++x) { DesktopVector p(x, y); if (state == MouseCursorMonitor::INSIDE && image_rect.Contains(p)) { EXPECT_EQ(BlendPixels(GetFakeFramePixelValue(p), kTestCursorData[y - pos.y()][x - pos.x()]), GetFramePixel(frame, p)); } else { EXPECT_EQ(GetFakeFramePixelValue(p), GetFramePixel(frame, p)); } } } } } // namespace bool operator==(const DesktopRect& left, const DesktopRect& right) { return left.equals(right); } std::ostream& operator<<(std::ostream& out, const DesktopRect& rect) { out << "{" << rect.left() << "+" << rect.top() << "-" << rect.width() << "x" << rect.height() << "}"; return out; } class DesktopAndCursorComposerTest : public ::testing::Test, public DesktopCapturer::Callback { public: explicit DesktopAndCursorComposerTest(bool include_cursor = true) : fake_screen_(new FakeScreenCapturer()), fake_cursor_(include_cursor ? new FakeMouseMonitor() : nullptr), blender_(fake_screen_, fake_cursor_) { blender_.Start(this); } // DesktopCapturer::Callback interface void OnCaptureResult(DesktopCapturer::Result result, std::unique_ptr frame) override { frame_ = std::move(frame); } protected: // Owned by `blender_`. FakeScreenCapturer* fake_screen_; FakeMouseMonitor* fake_cursor_; DesktopAndCursorComposer blender_; std::unique_ptr frame_; }; class DesktopAndCursorComposerNoCursorMonitorTest : public DesktopAndCursorComposerTest { public: DesktopAndCursorComposerNoCursorMonitorTest() : DesktopAndCursorComposerTest(false) {} }; TEST_F(DesktopAndCursorComposerTest, CursorShouldBeIgnoredIfNoFrameCaptured) { struct { int x, y; int hotspot_x, hotspot_y; bool inside; } tests[] = { {0, 0, 0, 0, true}, {50, 50, 0, 0, true}, {100, 50, 0, 0, true}, {50, 100, 0, 0, true}, {100, 100, 0, 0, true}, {0, 0, 2, 5, true}, {1, 1, 2, 5, true}, {50, 50, 2, 5, true}, {100, 100, 2, 5, true}, {0, 0, 5, 2, true}, {50, 50, 5, 2, true}, {100, 100, 5, 2, true}, {0, 0, 0, 0, false}, }; for (size_t i = 0; i < arraysize(tests); i++) { SCOPED_TRACE(i); DesktopVector hotspot(tests[i].hotspot_x, tests[i].hotspot_y); fake_cursor_->SetHotspot(hotspot); MouseCursorMonitor::CursorState state = tests[i].inside ? MouseCursorMonitor::INSIDE : MouseCursorMonitor::OUTSIDE; DesktopVector pos(tests[i].x, tests[i].y); fake_cursor_->SetState(state, pos); std::unique_ptr frame( SharedDesktopFrame::Wrap(CreateTestFrame())); blender_.CaptureFrame(); // If capturer captured nothing, then cursor should be ignored, not matter // its state or position. EXPECT_EQ(frame_, nullptr); } } TEST_F(DesktopAndCursorComposerTest, CursorShouldBeIgnoredIfFrameMayContainIt) { // We can't use a shared frame because we need to detect modifications // compared to a control. std::unique_ptr control_frame(CreateTestFrame()); control_frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord)); struct { int x; int y; bool may_contain_cursor; } tests[] = { {100, 200, true}, {100, 200, false}, {150, 250, true}, {150, 250, false}, }; for (size_t i = 0; i < arraysize(tests); i++) { SCOPED_TRACE(i); std::unique_ptr frame(CreateTestFrame()); frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord)); frame->set_may_contain_cursor(tests[i].may_contain_cursor); fake_screen_->SetNextFrame(std::move(frame)); const DesktopVector abs_pos(tests[i].x, tests[i].y); fake_cursor_->SetState(MouseCursorMonitor::INSIDE, abs_pos); blender_.CaptureFrame(); // If the frame may already have contained the cursor, then `CaptureFrame()` // should not have modified it, so it should be the same as the control. EXPECT_TRUE(frame_); const DesktopVector rel_pos(abs_pos.subtract(control_frame->top_left())); if (tests[i].may_contain_cursor) { EXPECT_EQ( *reinterpret_cast(frame_->GetFrameDataAtPos(rel_pos)), *reinterpret_cast( control_frame->GetFrameDataAtPos(rel_pos))); } else { // `CaptureFrame()` should have modified the frame to have the cursor. EXPECT_NE( *reinterpret_cast(frame_->GetFrameDataAtPos(rel_pos)), *reinterpret_cast( control_frame->GetFrameDataAtPos(rel_pos))); EXPECT_TRUE(frame_->may_contain_cursor()); } } } TEST_F(DesktopAndCursorComposerTest, CursorShouldBeIgnoredIfItIsOutOfDesktopFrame) { std::unique_ptr frame( SharedDesktopFrame::Wrap(CreateTestFrame())); frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord)); // The frame covers (100, 200) - (200, 300). struct { int x; int y; } tests[] = { {0, 0}, {50, 50}, {50, 150}, {100, 150}, {50, 200}, {99, 200}, {100, 199}, {200, 300}, {200, 299}, {199, 300}, {-1, -1}, {-10000, -10000}, {10000, 10000}, }; for (size_t i = 0; i < arraysize(tests); i++) { SCOPED_TRACE(i); fake_screen_->SetNextFrame(frame->Share()); // The CursorState is ignored when using absolute cursor position. fake_cursor_->SetState(MouseCursorMonitor::OUTSIDE, DesktopVector(tests[i].x, tests[i].y)); blender_.CaptureFrame(); VerifyFrame(*frame_, MouseCursorMonitor::OUTSIDE, DesktopVector(0, 0)); } } TEST_F(DesktopAndCursorComposerTest, IsOccludedShouldBeConsidered) { std::unique_ptr frame( SharedDesktopFrame::Wrap(CreateTestFrame())); frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord)); // The frame covers (100, 200) - (200, 300). struct { int x; int y; } tests[] = { {100, 200}, {101, 200}, {100, 201}, {101, 201}, {150, 250}, {199, 299}, }; fake_screen_->set_is_occluded(true); for (size_t i = 0; i < arraysize(tests); i++) { SCOPED_TRACE(i); fake_screen_->SetNextFrame(frame->Share()); // The CursorState is ignored when using absolute cursor position. fake_cursor_->SetState(MouseCursorMonitor::OUTSIDE, DesktopVector(tests[i].x, tests[i].y)); blender_.CaptureFrame(); VerifyFrame(*frame_, MouseCursorMonitor::OUTSIDE, DesktopVector()); } } TEST_F(DesktopAndCursorComposerTest, CursorIncluded) { std::unique_ptr frame( SharedDesktopFrame::Wrap(CreateTestFrame())); frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord)); // The frame covers (100, 200) - (200, 300). struct { int x; int y; } tests[] = { {100, 200}, {101, 200}, {100, 201}, {101, 201}, {150, 250}, {199, 299}, }; for (size_t i = 0; i < arraysize(tests); i++) { SCOPED_TRACE(i); const DesktopVector abs_pos(tests[i].x, tests[i].y); const DesktopVector rel_pos(abs_pos.subtract(frame->top_left())); fake_screen_->SetNextFrame(frame->Share()); // The CursorState is ignored when using absolute cursor position. fake_cursor_->SetState(MouseCursorMonitor::OUTSIDE, abs_pos); blender_.CaptureFrame(); VerifyFrame(*frame_, MouseCursorMonitor::INSIDE, rel_pos); // Verify that the cursor is erased before the frame buffer is returned to // the screen capturer. frame_.reset(); VerifyFrame(*frame, MouseCursorMonitor::OUTSIDE, DesktopVector()); } } TEST_F(DesktopAndCursorComposerNoCursorMonitorTest, UpdatedRegionIncludesOldAndNewCursorRectsIfMoved) { std::unique_ptr frame( SharedDesktopFrame::Wrap(CreateTestFrame())); DesktopRect first_cursor_rect; { // Block to scope test_cursor, which is invalidated by OnMouseCursor. MouseCursor* test_cursor = CreateTestCursor(DesktopVector(0, 0)); first_cursor_rect = DesktopRect::MakeSize(test_cursor->image()->size()); blender_.OnMouseCursor(test_cursor); } blender_.OnMouseCursorPosition(DesktopVector(0, 0)); fake_screen_->SetNextFrame(frame->Share()); blender_.CaptureFrame(); DesktopVector cursor_move_offset(1, 1); DesktopRect second_cursor_rect = first_cursor_rect; second_cursor_rect.Translate(cursor_move_offset); blender_.OnMouseCursorPosition(cursor_move_offset); fake_screen_->SetNextFrame(frame->Share()); blender_.CaptureFrame(); EXPECT_TRUE(frame->updated_region().is_empty()); DesktopRegion expected_region; expected_region.AddRect(first_cursor_rect); expected_region.AddRect(second_cursor_rect); EXPECT_TRUE(frame_->updated_region().Equals(expected_region)); } TEST_F(DesktopAndCursorComposerNoCursorMonitorTest, UpdatedRegionIncludesOldAndNewCursorRectsIfShapeChanged) { std::unique_ptr frame( SharedDesktopFrame::Wrap(CreateTestFrame())); DesktopRect first_cursor_rect; { // Block to scope test_cursor, which is invalidated by OnMouseCursor. MouseCursor* test_cursor = CreateTestCursor(DesktopVector(0, 0)); first_cursor_rect = DesktopRect::MakeSize(test_cursor->image()->size()); blender_.OnMouseCursor(test_cursor); } blender_.OnMouseCursorPosition(DesktopVector(0, 0)); fake_screen_->SetNextFrame(frame->Share()); blender_.CaptureFrame(); // Create a second cursor, the same shape as the first. Since the code doesn't // compare the cursor pixels, this is sufficient, and avoids needing two test // cursor bitmaps. DesktopRect second_cursor_rect; { MouseCursor* test_cursor = CreateTestCursor(DesktopVector(0, 0)); second_cursor_rect = DesktopRect::MakeSize(test_cursor->image()->size()); blender_.OnMouseCursor(test_cursor); } fake_screen_->SetNextFrame(frame->Share()); blender_.CaptureFrame(); EXPECT_TRUE(frame->updated_region().is_empty()); DesktopRegion expected_region; expected_region.AddRect(first_cursor_rect); expected_region.AddRect(second_cursor_rect); EXPECT_TRUE(frame_->updated_region().Equals(expected_region)); } TEST_F(DesktopAndCursorComposerNoCursorMonitorTest, UpdatedRegionUnchangedIfCursorUnchanged) { std::unique_ptr frame( SharedDesktopFrame::Wrap(CreateTestFrame())); blender_.OnMouseCursor(CreateTestCursor(DesktopVector(0, 0))); blender_.OnMouseCursorPosition(DesktopVector(0, 0)); fake_screen_->SetNextFrame(frame->Share()); blender_.CaptureFrame(); fake_screen_->SetNextFrame(frame->Share()); blender_.CaptureFrame(); EXPECT_TRUE(frame->updated_region().is_empty()); EXPECT_TRUE(frame_->updated_region().is_empty()); } } // namespace webrtc