summaryrefslogtreecommitdiffstats
path: root/widget/cocoa/TextRecognition.mm
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--widget/cocoa/TextRecognition.mm109
1 files changed, 109 insertions, 0 deletions
diff --git a/widget/cocoa/TextRecognition.mm b/widget/cocoa/TextRecognition.mm
new file mode 100644
index 0000000000..1c962a2c05
--- /dev/null
+++ b/widget/cocoa/TextRecognition.mm
@@ -0,0 +1,109 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#import <Vision/Vision.h>
+
+#include "mozilla/dom/Promise.h"
+#include "mozilla/gfx/2D.h"
+#include "mozilla/ErrorResult.h"
+#include "ErrorList.h"
+#include "nsClipboard.h"
+#include "nsCocoaUtils.h"
+#include "mozilla/MacStringHelpers.h"
+#include "mozilla/ScopeExit.h"
+#include "mozilla/widget/TextRecognition.h"
+#include "mozilla/dom/PContent.h"
+
+namespace mozilla::widget {
+
+auto TextRecognition::DoFindText(gfx::DataSourceSurface& aSurface,
+ const nsTArray<nsCString>& aLanguages) -> RefPtr<NativePromise> {
+ NS_OBJC_BEGIN_TRY_IGNORE_BLOCK
+ if (@available(macOS 10.15, *)) {
+ // TODO - Is this the most efficient path? Maybe we can write a new
+ // CreateCGImageFromXXX that enables more efficient marshalling of the data.
+ CGImageRef imageRef = NULL;
+ nsresult rv = nsCocoaUtils::CreateCGImageFromSurface(&aSurface, &imageRef);
+ if (NS_FAILED(rv) || !imageRef) {
+ return NativePromise::CreateAndReject("Failed to create CGImage"_ns, __func__);
+ }
+
+ auto promise = MakeRefPtr<NativePromise::Private>(__func__);
+
+ NSMutableArray* recognitionLanguages = [[NSMutableArray alloc] init];
+ for (const auto& locale : aLanguages) {
+ [recognitionLanguages addObject:nsCocoaUtils::ToNSString(locale)];
+ }
+
+ NS_DispatchBackgroundTask(
+ NS_NewRunnableFunction(
+ __func__,
+ [promise, imageRef, recognitionLanguages] {
+ auto unrefImage = MakeScopeExit([&] {
+ ::CGImageRelease(imageRef);
+ [recognitionLanguages release];
+ });
+
+ dom::TextRecognitionResult result;
+ dom::TextRecognitionResult* pResult = &result;
+
+ // Define the request to use, which also handles the result. It will be run below
+ // directly in this thread. After creating this request.
+ VNRecognizeTextRequest* textRecognitionRequest = [[VNRecognizeTextRequest alloc]
+ initWithCompletionHandler:^(VNRequest* _Nonnull request,
+ NSError* _Nullable error) {
+ NSArray<VNRecognizedTextObservation*>* observations = request.results;
+
+ [observations
+ enumerateObjectsUsingBlock:^(VNRecognizedTextObservation* _Nonnull obj,
+ NSUInteger idx, BOOL* _Nonnull stop) {
+ // Requests the n top candidates for a recognized text string.
+ VNRecognizedText* recognizedText = [obj topCandidates:1].firstObject;
+
+ // https://developer.apple.com/documentation/vision/vnrecognizedtext?language=objc
+ auto& quad = *pResult->quads().AppendElement();
+ CopyCocoaStringToXPCOMString(recognizedText.string, quad.string());
+ quad.confidence() = recognizedText.confidence;
+
+ auto ToImagePoint = [](CGPoint aPoint) -> ImagePoint {
+ return {static_cast<float>(aPoint.x), static_cast<float>(aPoint.y)};
+ };
+ *quad.points().AppendElement() = ToImagePoint(obj.bottomLeft);
+ *quad.points().AppendElement() = ToImagePoint(obj.topLeft);
+ *quad.points().AppendElement() = ToImagePoint(obj.topRight);
+ *quad.points().AppendElement() = ToImagePoint(obj.bottomRight);
+ }];
+ }];
+
+ textRecognitionRequest.recognitionLevel = VNRequestTextRecognitionLevelAccurate;
+ textRecognitionRequest.recognitionLanguages = recognitionLanguages;
+ textRecognitionRequest.usesLanguageCorrection = true;
+
+ // Send out the request. This blocks execution of this thread with an expensive
+ // CPU call.
+ NSError* error = nil;
+ VNImageRequestHandler* requestHandler =
+ [[[VNImageRequestHandler alloc] initWithCGImage:imageRef
+ options:@{}] autorelease];
+
+ [requestHandler performRequests:@[ textRecognitionRequest ] error:&error];
+ if (error != nil) {
+ promise->Reject(
+ nsPrintfCString("Failed to perform text recognition request (%ld)\n",
+ error.code),
+ __func__);
+ } else {
+ promise->Resolve(std::move(result), __func__);
+ }
+ }),
+ NS_DISPATCH_EVENT_MAY_BLOCK);
+ return promise;
+ } else {
+ return NativePromise::CreateAndReject("Text recognition is not available"_ns, __func__);
+ }
+
+ NS_OBJC_END_TRY_IGNORE_BLOCK
+}
+
+} // namespace mozilla::widget