Discharge7443/brave-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduced copy text from image feature on macOS

Browse Source 
fix https://github.com/brave/brave-browser/issues/27513

"Copy Text From Image" entry is added to render view's context menu.
It gets bitmap data from renderer via LocalFrame::GetImage() mojom interface,
and then passed it to TextRecognitionDialog to extract and show text from that
image.
This commit is contained in:
Simon Hong
2023-01-13 19:22:05 +09:00
parent 65525e800c
commit 9cf444c6d8
25 changed files with 627 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/BUILD.gn
+2
View File
@@ -6,6 +6,7 @@
import("//brave/browser/shell_integrations/buildflags/buildflags.gni")
import("//brave/components/brave_vpn/common/buildflags/buildflags.gni")
import("//brave/components/speedreader/common/buildflags/buildflags.gni")
import("//brave/components/text_recognition/common/buildflags/buildflags.gni")
import("//brave/resources/brave_grit.gni")
import("//build/config/features.gni")
import("//build/config/locales.gni")
@@ -19,6 +20,7 @@ brave_grit("brave_generated_resources_grit") {
"enable_speedreader=$enable_speedreader",
"enable_brave_vpn=$enable_brave_vpn",
"enable_pin_shortcut=$enable_pin_shortcut",
"enable_text_recognition=$enable_text_recognition",
]
source = "brave_generated_resources.grd"
output_dir = "$root_gen_dir/brave"
app/brave_command_ids.h
+1
View File
@@ -51,6 +51,7 @@
#define IDC_COPY_CLEAN_LINK 56040
#define IDC_TOGGLE_TAB_MUTE 56041
#define IDC_SIDEBAR_TOGGLE_POSITION 56042
#define IDC_CONTENT_CONTEXT_COPY_TEXT_FROM_IMAGE 56043
#define IDC_CONTENT_CONTEXT_IMPORT_IPNS_KEYS_START 56100
#define IDC_CONTENT_CONTEXT_IMPORT_IPNS_KEYS_END 56199
app/brave_generated_resources.grd
+19
View File
@@ -896,6 +896,25 @@ Or change later at <ph name="SETTINGS_EXTENIONS_LINK">$2<ex>brave://settings/ext
</message>
</if>
<!-- Text recognition dialog-->
<if expr="enable_text_recognition">
<message name="IDS_CONTENT_CONTEXT_COPY_TEXT_FROM_IMAGE" desc="The name of the context menu item to get text from image">
Copy Text From Image
</message>
<message name="IDS_TEXT_RECOG_DIALOG_CLOSE_BUTTON" desc="The text for the close button of text recognition dialog">
Close
</message>
<message name="IDS_TEXT_RECOG_DIALOG_HEADER_IN_PROGRESS" desc="The text for the header of text recognition dialog">
Copying text from image...
</message>
<message name="IDS_TEXT_RECOG_DIALOG_HEADER_COMPLETE" desc="The text for the header of text recognition dialog">
Text copied from image
</message>
<message name="IDS_TEXT_RECOG_DIALOG_HEADER_FAILED" desc="The text for the header of text recognition dialog">
Text copy failed
</message>
</if>
<!-- Brave VPN -->
<if expr="enable_brave_vpn">
<message name="IDS_SETTINGS_SHOW_VPN_BUTTON" desc="The label for showing vpn button in the toolbar">
browser/ui/BUILD.gn
+10
View File
@@ -12,6 +12,7 @@ import("//brave/components/ipfs/buildflags/buildflags.gni")
import("//brave/components/ntp_background_images/buildflags/buildflags.gni")
import("//brave/components/playlist/common/buildflags/buildflags.gni")
import("//brave/components/speedreader/common/buildflags/buildflags.gni")
import("//brave/components/text_recognition/common/buildflags/buildflags.gni")
import("//brave/components/tor/buildflags/buildflags.gni")
import("//build/config/features.gni")
import("//chrome/common/features.gni")
@@ -443,6 +444,15 @@ source_set("ui") {
]
}
if (enable_text_recognition) {
sources += [
"views/text_recognition_dialog_view.cc",
"views/text_recognition_dialog_view.h",
]
deps += [ "//brave/components/text_recognition/browser" ]
}
# This is no longer compiled into Chromium on Android, but we still
# need it
if (is_android) {
browser/ui/browser_dialogs.h
+9
View File
@@ -9,6 +9,11 @@
#include "base/callback_forward.h"
class Browser;
class SkBitmap;
namespace content {
class WebContents;
} // namespace content
namespace brave {
@@ -18,6 +23,10 @@ void ShowCrashReportPermissionAskDialog(Browser* browser);
// Run |callback| when dialog closed.
void ShowObsoleteSystemConfirmDialog(base::OnceCallback<void(bool)> callback);
// Show web modal dialog for showing text that recognized from |image|.
void ShowTextRecognitionDialog(content::WebContents* web_contents,
const SkBitmap& image);
} // namespace brave
#endif // BRAVE_BROWSER_UI_BROWSER_DIALOGS_H_
browser/ui/views/text_recognition_dialog_view.cc
+126
View File
@@ -0,0 +1,126 @@
/* Copyright (c) 2023 The Brave Authors. All rights reserved.
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "brave/browser/ui/views/text_recognition_dialog_view.h"
#include <memory>
#include "base/bind.h"
#include "base/strings/utf_string_conversions.h"
#include "base/task/task_traits.h"
#include "base/task/thread_pool.h"
#include "brave/components/l10n/common/localization_util.h"
#include "brave/components/text_recognition/browser/text_recognition.h"
#include "brave/grit/brave_generated_resources.h"
#include "components/constrained_window/constrained_window_views.h"
#include "ui/base/clipboard/scoped_clipboard_writer.h"
#include "ui/base/metadata/metadata_impl_macros.h"
#include "ui/gfx/geometry/insets.h"
#include "ui/views/controls/label.h"
#include "ui/views/controls/scroll_view.h"
#include "ui/views/layout/flex_layout.h"
#include "ui/views/widget/widget.h"
#include "ui/views/window/dialog_client_view.h"
namespace brave {
void ShowTextRecognitionDialog(content::WebContents* web_contents,
const SkBitmap& image) {
constrained_window::ShowWebModalDialogViews(
new TextRecognitionDialogView(image), web_contents)
->Show();
}
} // namespace brave
TextRecognitionDialogView::TextRecognitionDialogView(const SkBitmap& image) {
SetModalType(ui::MODAL_TYPE_CHILD);
SetButtons(ui::DIALOG_BUTTON_OK);
SetButtonLabel(ui::DIALOG_BUTTON_OK,
brave_l10n::GetLocalizedResourceUTF16String(
IDS_TEXT_RECOG_DIALOG_CLOSE_BUTTON));
SetShowCloseButton(false);
SetLayoutManager(std::make_unique<views::FlexLayout>())
->SetOrientation(views::LayoutOrientation::kVertical)
.SetMainAxisAlignment(views::LayoutAlignment::kStart)
.SetInteriorMargin(gfx::Insets::TLBR(24, 26, 0, 26));
header_label_ = AddChildView(std::make_unique<views::Label>());
const int size_diff = 14 - views::Label::GetDefaultFontList().GetFontSize();
header_label_->SetFontList(
views::Label::GetDefaultFontList()
.DeriveWithSizeDelta(size_diff)
.DeriveWithWeight(gfx::Font::Weight::SEMIBOLD));
header_label_->SetHorizontalAlignment(gfx::ALIGN_LEFT);
header_label_->SetProperty(views::kMarginsKey,
gfx::Insets::TLBR(0, 0, 10, 0));
if (image.empty()) {
UpdateContents({});
return;
}
StartExtractingText(image);
}
TextRecognitionDialogView::~TextRecognitionDialogView() = default;
void TextRecognitionDialogView::StartExtractingText(const SkBitmap& image) {
header_label_->SetText(brave_l10n::GetLocalizedResourceUTF16String(
IDS_TEXT_RECOG_DIALOG_HEADER_IN_PROGRESS));
base::ThreadPool::PostTaskAndReplyWithResult(
FROM_HERE,
{base::MayBlock(), base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN},
base::BindOnce(&GetTextFromImage, image),
base::BindOnce(&TextRecognitionDialogView::OnGetTextFromImage,
weak_factory_.GetWeakPtr()));
}
void TextRecognitionDialogView::OnGetTextFromImage(
const std::vector<std::string>& text) {
UpdateContents(text);
AdjustWidgetSize();
}
void TextRecognitionDialogView::UpdateContents(
const std::vector<std::string>& text) {
if (text.empty()) {
header_label_->SetText(brave_l10n::GetLocalizedResourceUTF16String(
IDS_TEXT_RECOG_DIALOG_HEADER_FAILED));
return;
}
header_label_->SetText(brave_l10n::GetLocalizedResourceUTF16String(
IDS_TEXT_RECOG_DIALOG_HEADER_COMPLETE));
// Treat each string in |text| as a separated line string.
std::string unified_string;
for (const auto& t : text) {
unified_string += t;
unified_string += '\n';
}
ui::ScopedClipboardWriter(ui::ClipboardBuffer::kCopyPaste)
.WriteText(base::UTF8ToUTF16(unified_string));
auto* scroll_view = AddChildView(std::make_unique<views::ScrollView>());
scroll_view->SetProperty(views::kMarginsKey, gfx::Insets::VH(0, 10));
scroll_view->ClipHeightTo(0, 350);
auto* label = scroll_view->SetContents(
std::make_unique<views::Label>(base::UTF8ToUTF16(unified_string)));
label->SetHorizontalAlignment(gfx::ALIGN_LEFT);
label->SetSelectable(true);
label->SetMultiLine(true);
}
void TextRecognitionDialogView::AdjustWidgetSize() {
GetWidget()->SetSize(GetDialogClientView()->GetPreferredSize());
}
BEGIN_METADATA(TextRecognitionDialogView, views::DialogDelegateView)
END_METADATA
browser/ui/views/text_recognition_dialog_view.h
+45
View File
@@ -0,0 +1,45 @@
/* Copyright (c) 2023 The Brave Authors. All rights reserved.
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef BRAVE_BROWSER_UI_VIEWS_TEXT_RECOGNITION_DIALOG_VIEW_H_
#define BRAVE_BROWSER_UI_VIEWS_TEXT_RECOGNITION_DIALOG_VIEW_H_
#include <string>
#include <vector>
#include "base/memory/raw_ptr.h"
#include "base/memory/weak_ptr.h"
#include "ui/base/metadata/metadata_header_macros.h"
#include "ui/views/window/dialog_delegate.h"
class SkBitmap;
namespace views {
class Label;
} // namespace views
class TextRecognitionDialogView : public views::DialogDelegateView {
public:
METADATA_HEADER(TextRecognitionDialogView);
explicit TextRecognitionDialogView(const SkBitmap& image);
TextRecognitionDialogView(const TextRecognitionDialogView&) = delete;
TextRecognitionDialogView& operator=(const TextRecognitionDialogView&) =
delete;
~TextRecognitionDialogView() override;
private:
void StartExtractingText(const SkBitmap& image);
void OnGetTextFromImage(const std::vector<std::string>& text);
// Show |text| in this dialog and copy it to clipboard.
void UpdateContents(const std::vector<std::string>& text);
void AdjustWidgetSize();
raw_ptr<views::Label> header_label_ = nullptr;
base::WeakPtrFactory<TextRecognitionDialogView> weak_factory_{this};
};
#endif // BRAVE_BROWSER_UI_VIEWS_TEXT_RECOGNITION_DIALOG_VIEW_H_
chromium_src/chrome/browser/DEPS
+1
View File
@@ -32,6 +32,7 @@ include_rules = [
"+brave/components/privacy_sandbox",
"+brave/components/sidebar",
"+brave/components/sync",
"+brave/components/text_recognition",
"+brave/components/tor",
"+brave/components/translate",
"+brave/components/url_sanitizer",
chromium_src/chrome/browser/renderer_context_menu/render_view_context_menu.cc
+46 -3
View File
@@ -1,6 +1,7 @@
// Copyright 2018 The Brave Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
/* Copyright (c) 2018 The Brave Authors. All rights reserved.
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "chrome/browser/renderer_context_menu/render_view_context_menu.h"
@@ -9,6 +10,7 @@
#include "brave/browser/profiles/profile_util.h"
#include "brave/browser/renderer_context_menu/brave_spelling_options_submenu_observer.h"
#include "brave/browser/ui/browser_commands.h"
#include "brave/browser/ui/browser_dialogs.h"
#include "brave/components/ipfs/buildflags/buildflags.h"
#include "brave/components/tor/buildflags/buildflags.h"
#include "brave/grit/brave_theme_resources.h"
@@ -137,6 +139,16 @@ void OnTorProfileCreated(const GURL& link_url,
#endif
#if BUILDFLAG(ENABLE_TEXT_RECOGNITION)
void OnGetImage(base::WeakPtr<content::WebContents> web_contents,
const SkBitmap& image) {
if (!web_contents)
return;
brave::ShowTextRecognitionDialog(web_contents.get(), image);
}
#endif
} // namespace
BraveRenderViewContextMenu::BraveRenderViewContextMenu(
@@ -152,6 +164,10 @@ BraveRenderViewContextMenu::BraveRenderViewContextMenu(
bool BraveRenderViewContextMenu::IsCommandIdEnabled(int id) const {
switch (id) {
#if BUILDFLAG(ENABLE_TEXT_RECOGNITION)
case IDC_CONTENT_CONTEXT_COPY_TEXT_FROM_IMAGE:
return params_.has_image_contents;
#endif
case IDC_COPY_CLEAN_LINK:
return params_.link_url.is_valid();
case IDC_CONTENT_CONTEXT_FORCE_PASTE:
@@ -249,12 +265,27 @@ void BraveRenderViewContextMenu::ExecuteCommand(int id, int event_flags) {
base::BindRepeating(OnTorProfileCreated, params_.link_url,
HasAlreadyOpenedTorWindow(GetProfile())));
break;
#endif
#if BUILDFLAG(ENABLE_TEXT_RECOGNITION)
case IDC_CONTENT_CONTEXT_COPY_TEXT_FROM_IMAGE:
CopyTextFromImage();
break;
#endif
default:
RenderViewContextMenu_Chromium::ExecuteCommand(id, event_flags);
}
}
#if BUILDFLAG(ENABLE_TEXT_RECOGNITION)
void BraveRenderViewContextMenu::CopyTextFromImage() {
RenderFrameHost* frame_host = GetRenderFrameHost();
if (frame_host)
frame_host->GetImageAt(
params_.x, params_.y,
base::BindOnce(OnGetImage, source_web_contents_->GetWeakPtr()));
}
#endif
void BraveRenderViewContextMenu::AddSpellCheckServiceItem(bool is_checked) {
// Call our implementation, not the one in the base class.
// Assumption:
@@ -382,6 +413,18 @@ void BraveRenderViewContextMenu::InitMenu() {
IDC_CONTENT_CONTEXT_FORCE_PASTE,
IDS_CONTENT_CONTEXT_FORCE_PASTE);
}
#if BUILDFLAG(ENABLE_TEXT_RECOGNITION)
bool media_image = content_type_->SupportsGroup(
ContextMenuContentType::ITEM_GROUP_MEDIA_IMAGE);
if (media_image) {
index =
menu_model_.GetIndexOfCommandId(IDC_CONTENT_CONTEXT_COPYIMAGELOCATION);
DCHECK(index);
menu_model_.InsertItemWithStringIdAt(
index.value() + 1, IDC_CONTENT_CONTEXT_COPY_TEXT_FROM_IMAGE,
IDS_CONTENT_CONTEXT_COPY_TEXT_FROM_IMAGE);
}
#endif
#if BUILDFLAG(ENABLE_TOR)
// Add Open Link with Tor
chromium_src/chrome/browser/renderer_context_menu/render_view_context_menu.h
+9 -3
View File
@@ -1,11 +1,13 @@
// Copyright 2018 The Brave Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
/* Copyright (c) 2018 The Brave Authors. All rights reserved.
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef BRAVE_CHROMIUM_SRC_CHROME_BROWSER_RENDERER_CONTEXT_MENU_RENDER_VIEW_CONTEXT_MENU_H_
#define BRAVE_CHROMIUM_SRC_CHROME_BROWSER_RENDERER_CONTEXT_MENU_RENDER_VIEW_CONTEXT_MENU_H_
#include "brave/components/ipfs/buildflags/buildflags.h"
#include "brave/components/text_recognition/common/buildflags/buildflags.h"
#define BRAVE_RENDER_VIEW_CONTEXT_MENU_H_ \
private: \
@@ -53,6 +55,10 @@ class BraveRenderViewContextMenu : public RenderViewContextMenu_Chromium {
ui::SimpleMenuModel ipfs_submenu_model_;
#endif
#if BUILDFLAG(ENABLE_TEXT_RECOGNITION)
void CopyTextFromImage();
#endif
};
// Use our own subclass as the real RenderViewContextMenu.
chromium_src/chrome/browser/sources.gni
+2
View File
@@ -4,12 +4,14 @@
# You can obtain one at http://mozilla.org/MPL/2.0/.
import("//brave/components/brave_vpn/common/buildflags/buildflags.gni")
import("//brave/components/text_recognition/common/buildflags/buildflags.gni")
import("//build/config/ui.gni")
brave_chromium_src_chrome_browser_deps = [
"//base",
"//brave/components/brave_vpn/common/buildflags",
"//brave/components/playlist/common/buildflags",
"//brave/components/text_recognition/common/buildflags",
"//chrome/common:channel_info",
"//components/version_info",
]
chromium_src/content/browser/renderer_host/render_frame_host_impl.cc
+20
View File
@@ -0,0 +1,20 @@
/* Copyright (c) 2023 The Brave Authors. All rights reserved.
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "src/content/browser/renderer_host/render_frame_host_impl.cc"
namespace content {
void RenderFrameHostImpl::GetImageAt(
int x,
int y,
base::OnceCallback<void(const SkBitmap&)> callback) {
gfx::PointF point_in_view =
GetView()->TransformRootPointToViewCoordSpace(gfx::PointF(x, y));
GetAssociatedLocalFrame()->GetImageAt(
gfx::Point(point_in_view.x(), point_in_view.y()), std::move(callback));
}
} // namespace content
chromium_src/content/browser/renderer_host/render_frame_host_impl.h
+21
View File
@@ -0,0 +1,21 @@
/* Copyright (c) 2023 The Brave Authors. All rights reserved.
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef BRAVE_CHROMIUM_SRC_CONTENT_BROWSER_RENDERER_HOST_RENDER_FRAME_HOST_IMPL_H_
#define BRAVE_CHROMIUM_SRC_CONTENT_BROWSER_RENDERER_HOST_RENDER_FRAME_HOST_IMPL_H_
#include "content/public/browser/render_frame_host.h"
#include "third_party/blink/public/mojom/frame/frame.mojom.h"
#define CopyImageAt \
GetImageAt(int x, int y, base::OnceCallback<void(const SkBitmap&)> callback) \
override; \
void CopyImageAt
#include "src/content/browser/renderer_host/render_frame_host_impl.h"
#undef CopyImageAt
#endif // BRAVE_CHROMIUM_SRC_CONTENT_BROWSER_RENDERER_HOST_RENDER_FRAME_HOST_IMPL_H_
chromium_src/content/public/browser/render_frame_host.h
+20
View File
@@ -0,0 +1,20 @@
/* Copyright (c) 2023 The Brave Authors. All rights reserved.
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef BRAVE_CHROMIUM_SRC_CONTENT_PUBLIC_BROWSER_RENDER_FRAME_HOST_H_
#define BRAVE_CHROMIUM_SRC_CONTENT_PUBLIC_BROWSER_RENDER_FRAME_HOST_H_
class SkBitmap;
#define CopyImageAt \
GetImageAt(int x, int y, \
base::OnceCallback<void(const SkBitmap&)> callback) = 0; \
virtual void CopyImageAt
#include "src/content/public/browser/render_frame_host.h"
#undef CopyImageAt
#endif // BRAVE_CHROMIUM_SRC_CONTENT_PUBLIC_BROWSER_RENDER_FRAME_HOST_H_
chromium_src/third_party/blink/public/mojom/frame/frame.mojom
Vendored
+15
View File
@@ -0,0 +1,15 @@
// Copyright (c) 2023 The Brave Authors. All rights reserved.
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
module blink.mojom;
import "skia/public/mojom/bitmap.mojom";
import "ui/gfx/geometry/mojom/geometry.mojom";
[BraveExtend]
interface LocalFrame {
GetImageAt(gfx.mojom.Point window_point)
=> (skia.mojom.BitmapN32? bitmap_image);
};
chromium_src/third_party/blink/renderer/core/frame/local_frame.cc
Vendored
+79
View File
@@ -6,7 +6,11 @@
#include "third_party/blink/renderer/core/frame/local_frame.h"
#include "brave/components/brave_page_graph/common/buildflags.h"
#include "skia/ext/skia_utils_base.h"
#include "third_party/blink/renderer/core/html/canvas/html_canvas_element.h"
#include "third_party/blink/renderer/core/layout/layout_image.h"
#include "third_party/blink/renderer/core/probe/core_probes.h"
#include "third_party/blink/renderer/platform/graphics/graphics_types_3d.h"
#if BUILDFLAG(ENABLE_BRAVE_PAGE_GRAPH)
#include "brave/third_party/blink/renderer/core/brave_page_graph/page_graph.h"
@@ -23,4 +27,79 @@
#include "src/third_party/blink/renderer/core/frame/local_frame.cc"
namespace blink {
namespace {
static scoped_refptr<Image> ImageFromNode(const Node& node) {
DCHECK(!node.GetDocument().NeedsLayoutTreeUpdate());
DocumentLifecycle::DisallowTransitionScope disallow_transition(
node.GetDocument().Lifecycle());
const LayoutObject* const layout_object = node.GetLayoutObject();
if (!layout_object)
return nullptr;
if (layout_object->IsCanvas()) {
return To<HTMLCanvasElement>(const_cast<Node&>(node))
.Snapshot(kFrontBuffer);
}
if (!layout_object->IsImage())
return nullptr;
const auto& layout_image = To<LayoutImage>(*layout_object);
const ImageResourceContent* const cached_image = layout_image.CachedImage();
if (!cached_image || cached_image->ErrorOccurred())
return nullptr;
return cached_image->GetImage();
}
} // namespace
SkBitmap LocalFrame::GetImageAtViewportPoint(const gfx::Point& viewport_point) {
HitTestResult result = HitTestResultForVisualViewportPos(viewport_point);
if (!IsA<HTMLCanvasElement>(result.InnerNodeOrImageMapImage()) &&
result.AbsoluteImageURL().IsEmpty()) {
// There isn't actually an image at these coordinates. Might be because
// the window scrolled while the context menu was open or because the page
// changed itself between when we thought there was an image here and when
// we actually tried to retrieve the image.
//
// FIXME: implement a cache of the most recent HitTestResult to avoid having
// to do two hit tests.
return {};
}
const scoped_refptr<Image> image =
ImageFromNode(*result.InnerNodeOrImageMapImage());
if (!image.get())
return {};
PaintImage paint_image = image->PaintImageForCurrentFrame();
// Orient the data.
if (!image->HasDefaultOrientation()) {
paint_image = Image::ResizeAndOrientImage(
paint_image, image->CurrentFrameOrientation(), gfx::Vector2dF(1, 1), 1,
kInterpolationNone);
}
SkBitmap bitmap;
if (sk_sp<SkImage> sk_image = paint_image.GetSwSkImage())
sk_image->asLegacyBitmap(&bitmap);
// The bitmap backing a canvas can be in non-native skia pixel order (aka
// RGBA when kN32_SkColorType is BGRA-ordered, or higher bit-depth color-types
// like F16. The IPC to the browser requires the bitmap to be in N32 format
// so we convert it here if needed.
SkBitmap n32_bitmap;
if (skia::SkBitmapToN32OpaqueOrPremul(bitmap, &n32_bitmap) &&
!n32_bitmap.isNull()) {
return n32_bitmap;
}
return {};
}
} // namespace blink
#undef FrameAttachedToParent
chromium_src/third_party/blink/renderer/core/frame/local_frame.h
Vendored
+20
View File
@@ -0,0 +1,20 @@
/* Copyright (c) 2023 The Brave Authors. All rights reserved.
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef BRAVE_CHROMIUM_SRC_THIRD_PARTY_BLINK_RENDERER_CORE_FRAME_LOCAL_FRAME_H_
#define BRAVE_CHROMIUM_SRC_THIRD_PARTY_BLINK_RENDERER_CORE_FRAME_LOCAL_FRAME_H_
class SkBitmap;
#define CopyImageAtViewportPoint \
CopyImageAtViewportPoint_UnUsed() {} \
SkBitmap GetImageAtViewportPoint(const gfx::Point& viewport_point); \
void CopyImageAtViewportPoint
#include "src/third_party/blink/renderer/core/frame/local_frame.h"
#undef CopyImageAtViewportPoint
#endif // BRAVE_CHROMIUM_SRC_THIRD_PARTY_BLINK_RENDERER_CORE_FRAME_LOCAL_FRAME_H_
chromium_src/third_party/blink/renderer/core/frame/local_frame_mojo_handler.cc
Vendored
+17
View File
@@ -0,0 +1,17 @@
/* Copyright (c) 2023 The Brave Authors. All rights reserved.
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "src/third_party/blink/renderer/core/frame/local_frame_mojo_handler.cc"
namespace blink {
void LocalFrameMojoHandler::GetImageAt(const gfx::Point& window_point,
GetImageAtCallback callback) {
gfx::Point viewport_position =
frame_->GetWidgetForLocalRoot()->DIPsToRoundedBlinkSpace(window_point);
std::move(callback).Run(frame_->GetImageAtViewportPoint(viewport_position));
}
} // namespace blink
chromium_src/third_party/blink/renderer/core/frame/local_frame_mojo_handler.h
Vendored
+20
View File
@@ -0,0 +1,20 @@
/* Copyright (c) 2023 The Brave Authors. All rights reserved.
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef BRAVE_CHROMIUM_SRC_THIRD_PARTY_BLINK_RENDERER_CORE_FRAME_LOCAL_FRAME_MOJO_HANDLER_H_
#define BRAVE_CHROMIUM_SRC_THIRD_PARTY_BLINK_RENDERER_CORE_FRAME_LOCAL_FRAME_MOJO_HANDLER_H_
#include "third_party/blink/public/mojom/frame/frame.mojom-blink.h"
#define CopyImageAt \
GetImageAt(const gfx::Point& window_point, GetImageAtCallback callback) \
final; \
void CopyImageAt
#include "src/third_party/blink/renderer/core/frame/local_frame_mojo_handler.h"
#undef CopyImageAt
#endif // BRAVE_CHROMIUM_SRC_THIRD_PARTY_BLINK_RENDERER_CORE_FRAME_LOCAL_FRAME_MOJO_HANDLER_H_
components/text_recognition/DEPS
+5
View File
@@ -0,0 +1,5 @@
include_rules = [
"+base",
"+skia",
"+third_party/skia",
]
components/text_recognition/browser/BUILD.gn
+28
View File
@@ -0,0 +1,28 @@
# Copyright (c) 2023 The Brave Authors. All rights reserved.
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at https://mozilla.org/MPL/2.0/.
import("//brave/components/text_recognition/common/buildflags/buildflags.gni")
assert(enable_text_recognition)
source_set("browser") {
sources = [ "text_recognition.h" ]
if (is_mac) {
sources += [ "text_recognition.mm" ]
frameworks = [
"Foundation.framework",
"Vision.framework",
]
}
deps = [
"//base",
"//skia",
]
configs += [ "//build/config/compiler:enable_arc" ]
}
components/text_recognition/browser/text_recognition.h
+17
View File
@@ -0,0 +1,17 @@
/* Copyright (c) 2023 The Brave Authors. All rights reserved.
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at https://mozilla.org/MPL/2.0/. */
#ifndef BRAVE_COMPONENTS_TEXT_RECOGNITION_BROWSER_TEXT_RECOGNITION_H_
#define BRAVE_COMPONENTS_TEXT_RECOGNITION_BROWSER_TEXT_RECOGNITION_H_
#include <string>
#include <vector>
class SkBitmap;
// Returns recognized texts from |image|.
std::vector<std::string> GetTextFromImage(const SkBitmap& image);
#endif // BRAVE_COMPONENTS_TEXT_RECOGNITION_BROWSER_TEXT_RECOGNITION_H_
components/text_recognition/browser/text_recognition.mm
+75
View File
@@ -0,0 +1,75 @@
/* Copyright (c) 2023 The Brave Authors. All rights reserved.
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at https://mozilla.org/MPL/2.0/. */
#include "brave/components/text_recognition/browser/text_recognition.h"
#import <Foundation/Foundation.h>
#import <Vision/Vision.h>
#include "base/logging.h"
#include "base/mac/foundation_util.h"
#include "base/mac/mac_util.h"
#include "base/mac/scoped_cftyperef.h"
#include "base/strings/sys_string_conversions.h"
#include "skia/ext/skia_utils_base.h"
#include "skia/ext/skia_utils_mac.h"
#include "third_party/skia/include/core/SkBitmap.h"
#include "third_party/skia/include/utils/mac/SkCGUtils.h"
std::vector<std::string> GetTextFromImage(const SkBitmap& image) {
std::vector<std::string> result;
if (@available(macOS 10.15, *)) {
// The bitmap type is sanitized to be N32 before we get here. The conversion
// to an NSImage would not explode if we got this wrong, so this is not a
// security CHECK.
DCHECK_EQ(image.colorType(), kN32_SkColorType);
auto* p_result = &result;
VNRecognizeTextRequest* textRecognitionRequest =
[[VNRecognizeTextRequest alloc] initWithCompletionHandler:^(
VNRequest* _Nonnull request,
NSError* _Nullable error) {
NSArray<VNRecognizedTextObservation*>* observations = request.results;
[observations enumerateObjectsUsingBlock:^(
VNRecognizedTextObservation* _Nonnull obj,
NSUInteger idx, BOOL* _Nonnull stop) {
// Ask first top candidate for a recognized text string.
VNRecognizedText* recognizedText =
[obj topCandidates:1].firstObject;
p_result->push_back(
base::SysNSStringToUTF8([recognizedText string]));
}];
}];
textRecognitionRequest.recognitionLevel =
VNRequestTextRecognitionLevelAccurate;
textRecognitionRequest.usesLanguageCorrection = true;
// Copied FF's supported language list.
// See https://support.mozilla.org/en-US/kb/text-recognition
// English, Chinese, Portuguese, French, Italian, German, and Spanish
textRecognitionRequest.recognitionLanguages = @[
@"en-US", @"zh-Hans", @"zh-Hant", @"pt-BR", @"fr-FR", @"it-IT", @"de-DE",
@"es-ES"
];
if (@available(macOS 13.0, *)) {
textRecognitionRequest.automaticallyDetectsLanguage = true;
}
NSError* error = nil;
base::ScopedCFTypeRef<CGImageRef> cg_image(SkCreateCGImageRef(image));
VNImageRequestHandler* requestHandler =
[[VNImageRequestHandler alloc] initWithCGImage:cg_image.get()
options:@{}];
[requestHandler performRequests:@[ textRecognitionRequest ] error:&error];
if (error) {
LOG(ERROR) << base::SysNSStringToUTF8([error localizedDescription]);
}
}
return result;
}
components/text_recognition/common/buildflags/BUILD.gn
+12
View File
@@ -0,0 +1,12 @@
# Copyright (c) 2023 The Brave Authors. All rights reserved.
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at https://mozilla.org/MPL/2.0/.
import("//brave/components/text_recognition/common/buildflags/buildflags.gni")
import("//build/buildflag_header.gni")
buildflag_header("buildflags") {
header = "buildflags.h"
flags = [ "ENABLE_TEXT_RECOGNITION=$enable_text_recognition" ]
}
components/text_recognition/common/buildflags/buildflags.gni
+8
View File
@@ -0,0 +1,8 @@
# Copyright (c) 2023 The Brave Authors. All rights reserved.
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at https://mozilla.org/MPL/2.0/.
declare_args() {
enable_text_recognition = is_mac
}