Discharge7443/brave-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Address previous WDP feedback

Browse Source
This commit is contained in:
Darnell Andries
2025-01-29 17:30:16 -08:00
parent 765483c1f6
commit f2bd96ba9c
8 changed files with 272 additions and 1056 deletions
Download Patch File Download Diff File Expand all files Collapse all files
browser/web_discovery/content_scraper_browsertest.cc
+1 -1
View File
@@ -164,7 +164,7 @@ class WebDiscoveryContentScraperTest : public PlatformBrowserTest {
server_config_loader_->SetLastPatternsForTesting(std::move(patterns_group));
scraper_ = std::make_unique<ContentScraper>(server_config_loader_.get());
scraper_ = ContentScraper::Create(server_config_loader_.get());
}
content::ContentMockCertVerifier mock_cert_verifier_;
chromium_src/chrome/renderer/DEPS
-3
View File
@@ -9,8 +9,5 @@ include_rules += [
"+brave/components/content_settings/renderer",
"+brave/components/tor/buildflags",
"+brave/components/tor/renderer",
"+brave/components/web_discovery/buildflags",
"+brave/components/web_discovery/common",
"+brave/components/web_discovery/renderer",
"+brave/renderer",
]
components/web_discovery/browser/content_scraper.cc
+256 -235
View File
@@ -13,9 +13,11 @@
#include "base/strings/string_split.h"
#include "base/task/task_traits.h"
#include "base/task/thread_pool.h"
#include "brave/components/web_discovery/browser/document_extractor/lib.rs.h"
#include "brave/components/web_discovery/browser/patterns.h"
#include "brave/components/web_discovery/browser/privacy_guard.h"
#include "brave/components/web_discovery/browser/util.h"
#include "third_party/rust/cxx/v1/cxx.h"
namespace web_discovery {
@@ -87,20 +89,266 @@ std::optional<std::string> RefineJsonExtract(const std::string& value,
return encoded_value;
}
class ContentScraperImpl : public ContentScraper {
public:
explicit ContentScraperImpl(const ServerConfigLoader* server_config_loader)
: sequenced_task_runner_(base::ThreadPool::CreateSequencedTaskRunner(
{base::TaskPriority::BEST_EFFORT,
base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN})),
server_config_loader_(server_config_loader) {}
~ContentScraperImpl() override = default;
ContentScraperImpl(const ContentScraperImpl&) = delete;
ContentScraperImpl& operator=(const ContentScraperImpl&) = delete;
void ScrapePage(const GURL& url,
bool is_strict_scrape,
mojom::DocumentExtractor* document_extractor,
PageScrapeResultCallback callback) override {
const auto* url_details =
server_config_loader_->GetLastPatterns().GetMatchingURLPattern(
url, is_strict_scrape);
if (!url_details) {
return;
}
auto interim_result =
std::make_unique<PageScrapeResult>(url, url_details->id);
std::vector<mojom::SelectRequestPtr> select_requests;
for (const auto& [selector, group] : url_details->scrape_rule_groups) {
auto select_request = mojom::SelectRequest::New();
select_request->root_selector = selector;
for (const auto& [report_key, rule] : group) {
if (rule->rule_type == ScrapeRuleType::kStandard) {
ProcessStandardRule(report_key, *rule, selector, url,
interim_result.get());
continue;
}
auto attribute_request = mojom::SelectAttributeRequest::New();
attribute_request->sub_selector = rule->sub_selector;
attribute_request->attribute = rule->attribute;
attribute_request->key = report_key;
select_request->attribute_requests.push_back(
std::move(attribute_request));
}
select_requests.push_back(std::move(select_request));
}
document_extractor->QueryElementAttributes(
std::move(select_requests),
base::BindOnce(&ContentScraperImpl::OnRendererScrapedElementAttributes,
base::Unretained(this), is_strict_scrape,
std::move(interim_result), std::move(callback)));
}
void ParseAndScrapePage(const GURL& url,
bool is_strict_scrape,
std::unique_ptr<PageScrapeResult> prev_result,
std::string html,
PageScrapeResultCallback callback) override {
const auto* url_details =
server_config_loader_->GetLastPatterns().GetMatchingURLPattern(
url, is_strict_scrape);
if (!url_details) {
return;
}
auto interim_result = std::move(prev_result);
std::vector<SelectRequest> select_requests;
for (const auto& [selector, group] : url_details->scrape_rule_groups) {
SelectRequest select_request;
select_request.root_selector = selector;
for (const auto& [report_key, rule] : group) {
if (rule->rule_type == ScrapeRuleType::kStandard) {
ProcessStandardRule(report_key, *rule, selector, url,
interim_result.get());
continue;
}
SelectAttributeRequest attribute_request{
.sub_selector = rule->sub_selector.value_or(""),
.key = report_key,
.attribute = rule->attribute,
};
select_request.attribute_requests.push_back(
std::move(attribute_request));
}
select_requests.push_back(std::move(select_request));
}
sequenced_task_runner_->PostTaskAndReplyWithResult(
FROM_HERE,
base::BindOnce(&query_element_attributes, html, select_requests),
base::BindOnce(&ContentScraperImpl::OnBrowserParsedElementAttributes,
weak_ptr_factory_.GetWeakPtr(), is_strict_scrape,
std::move(interim_result), std::move(callback)));
}
private:
void ProcessStandardRule(const std::string& report_key,
const ScrapeRule& rule,
const std::string& root_selector,
const GURL& url,
PageScrapeResult* scrape_result) {
std::string value;
if (rule.attribute == kUrlAttrId) {
value = url.spec();
} else if (rule.attribute == kCountryCodeAttrId) {
value = server_config_loader_->GetLastServerConfig().location;
}
auto refined_value = ExecuteRefineFunctions(rule.functions_applied, value);
if (!refined_value) {
return;
}
auto& fields = scrape_result->fields[root_selector];
if (fields.empty()) {
fields.emplace_back();
}
fields[0].Set(report_key, *refined_value);
}
void OnRendererScrapedElementAttributes(
bool is_strict_scrape,
std::unique_ptr<PageScrapeResult> scrape_result,
PageScrapeResultCallback callback,
std::vector<mojom::AttributeResultPtr> attribute_results) {
const auto* url_details =
server_config_loader_->GetLastPatterns().GetMatchingURLPattern(
scrape_result->url, is_strict_scrape);
if (!url_details) {
return;
}
for (const auto& attribute_result : attribute_results) {
const auto rule_group =
url_details->scrape_rule_groups.find(attribute_result->root_selector);
if (rule_group == url_details->scrape_rule_groups.end()) {
continue;
}
base::Value::Dict attribute_values;
for (const auto& [key, value_str] : attribute_result->attribute_values) {
ProcessAttributeValue(rule_group->second, *scrape_result, key,
value_str, attribute_values);
}
scrape_result->fields[attribute_result->root_selector].push_back(
std::move(attribute_values));
}
std::move(callback).Run(std::move(scrape_result));
}
void OnBrowserParsedElementAttributes(
bool is_strict_scrape,
std::unique_ptr<PageScrapeResult> scrape_result,
PageScrapeResultCallback callback,
rust::Vec<AttributeResult> attribute_results) {
const auto* url_details =
server_config_loader_->GetLastPatterns().GetMatchingURLPattern(
scrape_result->url, is_strict_scrape);
if (!url_details) {
return;
}
for (const auto& attribute_result : attribute_results) {
const auto root_selector = std::string(attribute_result.root_selector);
const auto rule_group =
url_details->scrape_rule_groups.find(root_selector);
if (rule_group == url_details->scrape_rule_groups.end()) {
continue;
}
base::Value::Dict attribute_values;
for (const auto& pair : attribute_result.attribute_pairs) {
ProcessAttributeValue(
rule_group->second, *scrape_result, std::string(pair.key),
pair.value.empty() ? std::nullopt
: std::make_optional<std::string>(pair.value),
attribute_values);
}
scrape_result->fields[root_selector].push_back(
std::move(attribute_values));
}
std::move(callback).Run(std::move(scrape_result));
}
std::optional<std::string> ExecuteRefineFunctions(
const RefineFunctionList& function_list,
std::string value) {
std::optional<std::string> result = value;
for (const auto& function_args : function_list) {
if (function_args.empty()) {
continue;
}
const auto& func_name = function_args[0];
if (func_name == kRefineSplitFuncId) {
if (function_args.size() >= 3 && function_args[1].is_string() &&
function_args[2].is_int()) {
result = RefineSplit(*result, function_args[1].GetString(),
function_args[2].GetInt());
}
} else if (func_name == kRefineMaskURLFuncId) {
result = MaskURL(GURL(value));
} else if (func_name == kRefineParseURLFuncId) {
if (function_args.size() >= 3 && function_args[1].is_string() &&
function_args[2].is_string()) {
result = RefineParseURL(*result, function_args[1].GetString(),
function_args[2].GetString());
}
} else if (func_name == kRefineJsonExtractFuncId) {
if (function_args.size() >= 3 && function_args[1].is_string() &&
function_args[2].is_bool()) {
result = RefineJsonExtract(*result, function_args[1].GetString(),
function_args[2].GetBool());
}
}
if (!result) {
break;
}
}
return result;
}
void ProcessAttributeValue(const ScrapeRuleGroup& rule_group,
PageScrapeResult& scrape_result,
std::string key,
std::optional<std::string> value_str,
base::Value::Dict& attribute_values) {
const auto rule = rule_group.find(key);
if (rule == rule_group.end()) {
return;
}
base::Value value;
if (value_str) {
value_str =
ExecuteRefineFunctions(rule->second->functions_applied, *value_str);
if (value_str) {
if (rule->second->rule_type == ScrapeRuleType::kSearchQuery ||
rule->second->rule_type == ScrapeRuleType::kWidgetTitle) {
scrape_result.query = value_str;
}
value = base::Value(*value_str);
}
}
attribute_values.Set(key, std::move(value));
}
scoped_refptr<base::SequencedTaskRunner> sequenced_task_runner_;
raw_ptr<const ServerConfigLoader> server_config_loader_;
base::WeakPtrFactory<ContentScraperImpl> weak_ptr_factory_{this};
};
} // namespace
std::unique_ptr<ContentScraper> ContentScraper::Create(
const ServerConfigLoader* server_config_loader) {
return std::make_unique<ContentScraperImpl>(server_config_loader);
}
PageScrapeResult::PageScrapeResult(GURL url, std::string id)
: url(url), id(id) {}
PageScrapeResult::~PageScrapeResult() = default;
ContentScraper::ContentScraper(const ServerConfigLoader* server_config_loader)
: sequenced_task_runner_(base::ThreadPool::CreateSequencedTaskRunner(
{base::TaskPriority::BEST_EFFORT,
base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN})),
server_config_loader_(server_config_loader) {}
ContentScraper::~ContentScraper() = default;
base::Value PageScrapeResult::SerializeToValue() {
base::Value::Dict result;
base::Value::Dict fields_dict;
@@ -151,232 +399,5 @@ std::unique_ptr<PageScrapeResult> PageScrapeResult::FromValue(
return result;
}
void ContentScraper::ScrapePage(const GURL& url,
bool is_strict_scrape,
mojom::DocumentExtractor* document_extractor,
PageScrapeResultCallback callback) {
const auto* url_details =
server_config_loader_->GetLastPatterns().GetMatchingURLPattern(
url, is_strict_scrape);
if (!url_details) {
return;
}
auto interim_result =
std::make_unique<PageScrapeResult>(url, url_details->id);
std::vector<mojom::SelectRequestPtr> select_requests;
for (const auto& [selector, group] : url_details->scrape_rule_groups) {
auto select_request = mojom::SelectRequest::New();
select_request->root_selector = selector;
for (const auto& [report_key, rule] : group) {
if (rule->rule_type == ScrapeRuleType::kStandard) {
ProcessStandardRule(report_key, *rule, selector, url,
interim_result.get());
continue;
}
auto attribute_request = mojom::SelectAttributeRequest::New();
attribute_request->sub_selector = rule->sub_selector;
attribute_request->attribute = rule->attribute;
attribute_request->key = report_key;
select_request->attribute_requests.push_back(
std::move(attribute_request));
}
select_requests.push_back(std::move(select_request));
}
document_extractor->QueryElementAttributes(
std::move(select_requests),
base::BindOnce(&ContentScraper::OnScrapedElementAttributes,
base::Unretained(this), is_strict_scrape,
std::move(interim_result), std::move(callback)));
}
void ContentScraper::ParseAndScrapePage(
const GURL& url,
bool is_strict_scrape,
std::unique_ptr<PageScrapeResult> prev_result,
std::string html,
PageScrapeResultCallback callback) {
const auto* url_details =
server_config_loader_->GetLastPatterns().GetMatchingURLPattern(
url, is_strict_scrape);
if (!url_details) {
return;
}
auto interim_result = std::move(prev_result);
std::vector<rust_document_extractor::SelectRequest> select_requests;
for (const auto& [selector, group] : url_details->scrape_rule_groups) {
rust_document_extractor::SelectRequest select_request;
select_request.root_selector = selector;
for (const auto& [report_key, rule] : group) {
if (rule->rule_type == ScrapeRuleType::kStandard) {
ProcessStandardRule(report_key, *rule, selector, url,
interim_result.get());
continue;
}
rust_document_extractor::SelectAttributeRequest attribute_request{
.sub_selector = rule->sub_selector.value_or(""),
.key = report_key,
.attribute = rule->attribute,
};
select_request.attribute_requests.push_back(std::move(attribute_request));
}
select_requests.push_back(std::move(select_request));
}
sequenced_task_runner_->PostTaskAndReplyWithResult(
FROM_HERE,
base::BindOnce(&rust_document_extractor::query_element_attributes, html,
select_requests),
base::BindOnce(&ContentScraper::OnRustElementAttributes,
weak_ptr_factory_.GetWeakPtr(), is_strict_scrape,
std::move(interim_result), std::move(callback)));
}
void ContentScraper::ProcessStandardRule(const std::string& report_key,
const ScrapeRule& rule,
const std::string& root_selector,
const GURL& url,
PageScrapeResult* scrape_result) {
std::string value;
if (rule.attribute == kUrlAttrId) {
value = url.spec();
} else if (rule.attribute == kCountryCodeAttrId) {
value = server_config_loader_->GetLastServerConfig().location;
}
auto refined_value = ExecuteRefineFunctions(rule.functions_applied, value);
if (!refined_value) {
return;
}
auto& fields = scrape_result->fields[root_selector];
if (fields.empty()) {
fields.emplace_back();
}
fields[0].Set(report_key, *refined_value);
}
void ContentScraper::OnScrapedElementAttributes(
bool is_strict_scrape,
std::unique_ptr<PageScrapeResult> scrape_result,
PageScrapeResultCallback callback,
std::vector<mojom::AttributeResultPtr> attribute_results) {
const auto* url_details =
server_config_loader_->GetLastPatterns().GetMatchingURLPattern(
scrape_result->url, is_strict_scrape);
if (!url_details) {
return;
}
for (const auto& attribute_result : attribute_results) {
const auto rule_group =
url_details->scrape_rule_groups.find(attribute_result->root_selector);
if (rule_group == url_details->scrape_rule_groups.end()) {
continue;
}
base::Value::Dict attribute_values;
for (const auto& [key, value_str] : attribute_result->attribute_values) {
ProcessAttributeValue(rule_group->second, *scrape_result, key, value_str,
attribute_values);
}
scrape_result->fields[attribute_result->root_selector].push_back(
std::move(attribute_values));
}
std::move(callback).Run(std::move(scrape_result));
}
void ContentScraper::OnRustElementAttributes(
bool is_strict_scrape,
std::unique_ptr<PageScrapeResult> scrape_result,
PageScrapeResultCallback callback,
rust::Vec<rust_document_extractor::AttributeResult> attribute_results) {
const auto* url_details =
server_config_loader_->GetLastPatterns().GetMatchingURLPattern(
scrape_result->url, is_strict_scrape);
if (!url_details) {
return;
}
for (const auto& attribute_result : attribute_results) {
const auto root_selector = std::string(attribute_result.root_selector);
const auto rule_group = url_details->scrape_rule_groups.find(root_selector);
if (rule_group == url_details->scrape_rule_groups.end()) {
continue;
}
base::Value::Dict attribute_values;
for (const auto& pair : attribute_result.attribute_pairs) {
ProcessAttributeValue(
rule_group->second, *scrape_result, std::string(pair.key),
pair.value.empty() ? std::nullopt
: std::make_optional<std::string>(pair.value),
attribute_values);
}
scrape_result->fields[root_selector].push_back(std::move(attribute_values));
}
std::move(callback).Run(std::move(scrape_result));
}
std::optional<std::string> ContentScraper::ExecuteRefineFunctions(
const RefineFunctionList& function_list,
std::string value) {
std::optional<std::string> result = value;
for (const auto& function_args : function_list) {
if (function_args.empty()) {
continue;
}
const auto& func_name = function_args[0];
if (func_name == kRefineSplitFuncId) {
if (function_args.size() >= 3 && function_args[1].is_string() &&
function_args[2].is_int()) {
result = RefineSplit(*result, function_args[1].GetString(),
function_args[2].GetInt());
}
} else if (func_name == kRefineMaskURLFuncId) {
result = MaskURL(GURL(value));
} else if (func_name == kRefineParseURLFuncId) {
if (function_args.size() >= 3 && function_args[1].is_string() &&
function_args[2].is_string()) {
result = RefineParseURL(*result, function_args[1].GetString(),
function_args[2].GetString());
}
} else if (func_name == kRefineJsonExtractFuncId) {
if (function_args.size() >= 3 && function_args[1].is_string() &&
function_args[2].is_bool()) {
result = RefineJsonExtract(*result, function_args[1].GetString(),
function_args[2].GetBool());
}
}
if (!result) {
break;
}
}
return result;
}
void ContentScraper::ProcessAttributeValue(
const ScrapeRuleGroup& rule_group,
PageScrapeResult& scrape_result,
std::string key,
std::optional<std::string> value_str,
base::Value::Dict& attribute_values) {
const auto rule = rule_group.find(key);
if (rule == rule_group.end()) {
return;
}
base::Value value;
if (value_str) {
value_str =
ExecuteRefineFunctions(rule->second->functions_applied, *value_str);
if (value_str) {
if (rule->second->rule_type == ScrapeRuleType::kSearchQuery ||
rule->second->rule_type == ScrapeRuleType::kWidgetTitle) {
scrape_result.query = value_str;
}
value = base::Value(*value_str);
}
}
attribute_values.Set(key, std::move(value));
}
} // namespace web_discovery
components/web_discovery/browser/content_scraper.h
+12 -47
View File
@@ -13,8 +13,6 @@
#include "base/containers/flat_map.h"
#include "base/functional/callback.h"
#include "base/values.h"
#include "brave/components/web_discovery/browser/document_extractor/lib.rs.h"
#include "brave/components/web_discovery/browser/patterns.h"
#include "brave/components/web_discovery/browser/server_config_loader.h"
#include "brave/components/web_discovery/common/web_discovery.mojom.h"
#include "url/gurl.h"
@@ -60,55 +58,22 @@ class ContentScraper {
using PageScrapeResultCallback =
base::OnceCallback<void(std::unique_ptr<PageScrapeResult>)>;
explicit ContentScraper(const ServerConfigLoader* server_config_loader);
~ContentScraper();
static std::unique_ptr<ContentScraper> Create(
const ServerConfigLoader* server_config_loader);
ContentScraper(const ContentScraper&) = delete;
ContentScraper& operator=(const ContentScraper&) = delete;
virtual ~ContentScraper() = default;
// For initial page scrape in renderer
void ScrapePage(const GURL& url,
bool is_strict_scrape,
mojom::DocumentExtractor* document_extractor,
PageScrapeResultCallback callback);
// For subsequent double fetches after initial scrape
void ParseAndScrapePage(const GURL& url,
virtual void ScrapePage(const GURL& url,
bool is_strict_scrape,
std::unique_ptr<PageScrapeResult> prev_result,
std::string html,
PageScrapeResultCallback callback);
private:
void ProcessStandardRule(const std::string& report_key,
const ScrapeRule& rule,
const std::string& root_selector,
const GURL& url,
PageScrapeResult* scrape_result);
void OnScrapedElementAttributes(
bool is_strict_scrape,
std::unique_ptr<PageScrapeResult> scrape_result,
PageScrapeResultCallback callback,
std::vector<mojom::AttributeResultPtr> attribute_results);
void OnRustElementAttributes(
bool is_strict_scrape,
std::unique_ptr<PageScrapeResult> scrape_result,
PageScrapeResultCallback callback,
rust::Vec<rust_document_extractor::AttributeResult> attribute_results);
std::optional<std::string> ExecuteRefineFunctions(
const RefineFunctionList& function_list,
std::string value);
void ProcessAttributeValue(const ScrapeRuleGroup& rule_group,
PageScrapeResult& scrape_result,
std::string key,
std::optional<std::string> value_str,
base::Value::Dict& attribute_values);
scoped_refptr<base::SequencedTaskRunner> sequenced_task_runner_;
raw_ptr<const ServerConfigLoader> server_config_loader_;
base::WeakPtrFactory<ContentScraper> weak_ptr_factory_{this};
mojom::DocumentExtractor* document_extractor,
PageScrapeResultCallback callback) = 0;
// For subsequent double fetches after initial scrape
virtual void ParseAndScrapePage(const GURL& url,
bool is_strict_scrape,
std::unique_ptr<PageScrapeResult> prev_result,
std::string html,
PageScrapeResultCallback callback) = 0;
};
} // namespace web_discovery
components/web_discovery/browser/document_extractor/.gitignore
+1
View File
@@ -1,2 +1,3 @@
target
example/main
Cargo.lock
components/web_discovery/browser/document_extractor/Cargo.lock
Generated
-767
View File
@@ -1,767 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "autocfg"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"
[[package]]
name = "byteorder"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "cc"
version = "1.0.98"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "convert_case"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
[[package]]
name = "cssparser"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a"
dependencies = [
"cssparser-macros",
"dtoa-short",
"itoa",
"matches",
"phf",
"proc-macro2",
"quote",
"smallvec",
"syn 1.0.109",
]
[[package]]
name = "cssparser-macros"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
dependencies = [
"quote",
"syn 2.0.66",
]
[[package]]
name = "cxx"
version = "1.0.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb497fad022245b29c2a0351df572e2d67c1046bcef2260ebc022aec81efea82"
dependencies = [
"cc",
"cxxbridge-flags",
"cxxbridge-macro",
"link-cplusplus",
]
[[package]]
name = "cxxbridge-flags"
version = "1.0.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "688c799a4a846f1c0acb9f36bb9c6272d9b3d9457f3633c7753c6057270df13c"
[[package]]
name = "cxxbridge-macro"
version = "1.0.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "928bc249a7e3cd554fd2e8e08a426e9670c50bbfc9a621653cfa9accc9641783"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
]
[[package]]
name = "derive_more"
version = "0.99.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321"
dependencies = [
"convert_case",
"proc-macro2",
"quote",
"rustc_version",
"syn 1.0.109",
]
[[package]]
name = "document-extractor-cxx"
version = "0.1.0"
dependencies = [
"cxx",
"html5ever",
"kuchikiki",
]
[[package]]
name = "dtoa"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653"
[[package]]
name = "dtoa-short"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbaceec3c6e4211c79e7b1800fb9680527106beb2f9c51904a3210c03a448c74"
dependencies = [
"dtoa",
]
[[package]]
name = "futf"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
dependencies = [
"mac",
"new_debug_unreachable",
]
[[package]]
name = "fxhash"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
dependencies = [
"byteorder",
]
[[package]]
name = "getrandom"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
dependencies = [
"cfg-if",
"libc",
"wasi 0.9.0+wasi-snapshot-preview1",
]
[[package]]
name = "getrandom"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
]
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "html5ever"
version = "0.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5c13fb08e5d4dfc151ee5e88bae63f7773d61852f3bdc73c9f4b9e1bde03148"
dependencies = [
"log",
"mac",
"markup5ever",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "indexmap"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown",
]
[[package]]
name = "itoa"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
name = "kuchikiki"
version = "0.8.4-speedreader"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af921794306a6885fc7510012ce12015d2cf0bfdba82fb217b9c2caf324a4618"
dependencies = [
"cssparser",
"html5ever",
"indexmap",
"matches",
"selectors",
]
[[package]]
name = "libc"
version = "0.2.155"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
[[package]]
name = "link-cplusplus"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9"
dependencies = [
"cc",
]
[[package]]
name = "lock_api"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
dependencies = [
"autocfg",
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
[[package]]
name = "mac"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "markup5ever"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd"
dependencies = [
"log",
"phf",
"phf_codegen",
"string_cache",
"string_cache_codegen",
"tendril",
]
[[package]]
name = "matches"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
[[package]]
name = "new_debug_unreachable"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
[[package]]
name = "nodrop"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
[[package]]
name = "once_cell"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "parking_lot"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-targets",
]
[[package]]
name = "phf"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12"
dependencies = [
"phf_macros",
"phf_shared 0.8.0",
"proc-macro-hack",
]
[[package]]
name = "phf_codegen"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815"
dependencies = [
"phf_generator 0.8.0",
"phf_shared 0.8.0",
]
[[package]]
name = "phf_generator"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526"
dependencies = [
"phf_shared 0.8.0",
"rand 0.7.3",
]
[[package]]
name = "phf_generator"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"
dependencies = [
"phf_shared 0.10.0",
"rand 0.8.5",
]
[[package]]
name = "phf_macros"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c"
dependencies = [
"phf_generator 0.8.0",
"phf_shared 0.8.0",
"proc-macro-hack",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "phf_shared"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7"
dependencies = [
"siphasher",
]
[[package]]
name = "phf_shared"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
dependencies = [
"siphasher",
]
[[package]]
name = "ppv-lite86"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]]
name = "precomputed-hash"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
[[package]]
name = "proc-macro-hack"
version = "0.5.20+deprecated"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
[[package]]
name = "proc-macro2"
version = "1.0.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
dependencies = [
"getrandom 0.1.16",
"libc",
"rand_chacha 0.2.2",
"rand_core 0.5.1",
"rand_hc",
"rand_pcg",
]
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha 0.3.1",
"rand_core 0.6.4",
]
[[package]]
name = "rand_chacha"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
dependencies = [
"ppv-lite86",
"rand_core 0.5.1",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core 0.6.4",
]
[[package]]
name = "rand_core"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
dependencies = [
"getrandom 0.1.16",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom 0.2.15",
]
[[package]]
name = "rand_hc"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
dependencies = [
"rand_core 0.5.1",
]
[[package]]
name = "rand_pcg"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"
dependencies = [
"rand_core 0.5.1",
]
[[package]]
name = "redox_syscall"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e"
dependencies = [
"bitflags 2.5.0",
]
[[package]]
name = "rustc_version"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
dependencies = [
"semver",
]
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "selectors"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe"
dependencies = [
"bitflags 1.3.2",
"cssparser",
"derive_more",
"fxhash",
"log",
"matches",
"phf",
"phf_codegen",
"precomputed-hash",
"servo_arc",
"smallvec",
"thin-slice",
]
[[package]]
name = "semver"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
[[package]]
name = "serde"
version = "1.0.203"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.203"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
]
[[package]]
name = "servo_arc"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432"
dependencies = [
"nodrop",
"stable_deref_trait",
]
[[package]]
name = "siphasher"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
[[package]]
name = "smallvec"
version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
[[package]]
name = "string_cache"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b"
dependencies = [
"new_debug_unreachable",
"once_cell",
"parking_lot",
"phf_shared 0.10.0",
"precomputed-hash",
"serde",
]
[[package]]
name = "string_cache_codegen"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988"
dependencies = [
"phf_generator 0.10.0",
"phf_shared 0.10.0",
"proc-macro2",
"quote",
]
[[package]]
name = "syn"
version = "1.0.109"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "syn"
version = "2.0.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "tendril"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"
dependencies = [
"futf",
"mac",
"utf-8",
]
[[package]]
name = "thin-slice"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"
[[package]]
name = "unicode-ident"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
[[package]]
name = "utf-8"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
[[package]]
name = "wasi"
version = "0.9.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "windows-targets"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
[[package]]
name = "windows_i686_gnu"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
[[package]]
name = "windows_i686_msvc"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
components/web_discovery/browser/document_extractor/lib.rs
+1 -1
View File
@@ -13,7 +13,7 @@ use kuchikiki::{
traits::TendrilSink,
};
#[cxx::bridge(namespace = "rust_document_extractor")]
#[cxx::bridge(namespace = "web_discovery")]
mod ffi {
pub struct SelectAttributeRequest {
/// An optional selector for an element within the current selected element.
components/web_discovery/browser/web_discovery_service.cc
+1 -2
View File
@@ -120,8 +120,7 @@ void WebDiscoveryService::OnConfigChange() {
void WebDiscoveryService::OnPatternsLoaded() {
if (!content_scraper_) {
content_scraper_ =
std::make_unique<ContentScraper>(server_config_loader_.get());
content_scraper_ = ContentScraper::Create(server_config_loader_.get());
}
if (!double_fetcher_) {
double_fetcher_ = std::make_unique<DoubleFetcher>(