diff --git a/browser/web_discovery/content_scraper_browsertest.cc b/browser/web_discovery/content_scraper_browsertest.cc index d2104448581..d761212220b 100644 --- a/browser/web_discovery/content_scraper_browsertest.cc +++ b/browser/web_discovery/content_scraper_browsertest.cc @@ -164,7 +164,7 @@ class WebDiscoveryContentScraperTest : public PlatformBrowserTest { server_config_loader_->SetLastPatternsForTesting(std::move(patterns_group)); - scraper_ = std::make_unique(server_config_loader_.get()); + scraper_ = ContentScraper::Create(server_config_loader_.get()); } content::ContentMockCertVerifier mock_cert_verifier_; diff --git a/chromium_src/chrome/renderer/DEPS b/chromium_src/chrome/renderer/DEPS index 3f59f093ad4..d4058040e92 100644 --- a/chromium_src/chrome/renderer/DEPS +++ b/chromium_src/chrome/renderer/DEPS @@ -9,8 +9,5 @@ include_rules += [ "+brave/components/content_settings/renderer", "+brave/components/tor/buildflags", "+brave/components/tor/renderer", - "+brave/components/web_discovery/buildflags", - "+brave/components/web_discovery/common", - "+brave/components/web_discovery/renderer", "+brave/renderer", ] diff --git a/components/web_discovery/browser/content_scraper.cc b/components/web_discovery/browser/content_scraper.cc index cd0ed6e9824..fa2e6d6f4c6 100644 --- a/components/web_discovery/browser/content_scraper.cc +++ b/components/web_discovery/browser/content_scraper.cc @@ -13,9 +13,11 @@ #include "base/strings/string_split.h" #include "base/task/task_traits.h" #include "base/task/thread_pool.h" +#include "brave/components/web_discovery/browser/document_extractor/lib.rs.h" #include "brave/components/web_discovery/browser/patterns.h" #include "brave/components/web_discovery/browser/privacy_guard.h" #include "brave/components/web_discovery/browser/util.h" +#include "third_party/rust/cxx/v1/cxx.h" namespace web_discovery { @@ -87,20 +89,266 @@ std::optional RefineJsonExtract(const std::string& value, return encoded_value; } +class ContentScraperImpl : public ContentScraper { + public: + explicit ContentScraperImpl(const ServerConfigLoader* server_config_loader) + : sequenced_task_runner_(base::ThreadPool::CreateSequencedTaskRunner( + {base::TaskPriority::BEST_EFFORT, + base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN})), + server_config_loader_(server_config_loader) {} + + ~ContentScraperImpl() override = default; + + ContentScraperImpl(const ContentScraperImpl&) = delete; + ContentScraperImpl& operator=(const ContentScraperImpl&) = delete; + + void ScrapePage(const GURL& url, + bool is_strict_scrape, + mojom::DocumentExtractor* document_extractor, + PageScrapeResultCallback callback) override { + const auto* url_details = + server_config_loader_->GetLastPatterns().GetMatchingURLPattern( + url, is_strict_scrape); + if (!url_details) { + return; + } + auto interim_result = + std::make_unique(url, url_details->id); + + std::vector select_requests; + for (const auto& [selector, group] : url_details->scrape_rule_groups) { + auto select_request = mojom::SelectRequest::New(); + select_request->root_selector = selector; + for (const auto& [report_key, rule] : group) { + if (rule->rule_type == ScrapeRuleType::kStandard) { + ProcessStandardRule(report_key, *rule, selector, url, + interim_result.get()); + continue; + } + auto attribute_request = mojom::SelectAttributeRequest::New(); + attribute_request->sub_selector = rule->sub_selector; + attribute_request->attribute = rule->attribute; + attribute_request->key = report_key; + + select_request->attribute_requests.push_back( + std::move(attribute_request)); + } + select_requests.push_back(std::move(select_request)); + } + + document_extractor->QueryElementAttributes( + std::move(select_requests), + base::BindOnce(&ContentScraperImpl::OnRendererScrapedElementAttributes, + base::Unretained(this), is_strict_scrape, + std::move(interim_result), std::move(callback))); + } + + void ParseAndScrapePage(const GURL& url, + bool is_strict_scrape, + std::unique_ptr prev_result, + std::string html, + PageScrapeResultCallback callback) override { + const auto* url_details = + server_config_loader_->GetLastPatterns().GetMatchingURLPattern( + url, is_strict_scrape); + if (!url_details) { + return; + } + auto interim_result = std::move(prev_result); + + std::vector select_requests; + for (const auto& [selector, group] : url_details->scrape_rule_groups) { + SelectRequest select_request; + select_request.root_selector = selector; + for (const auto& [report_key, rule] : group) { + if (rule->rule_type == ScrapeRuleType::kStandard) { + ProcessStandardRule(report_key, *rule, selector, url, + interim_result.get()); + continue; + } + SelectAttributeRequest attribute_request{ + .sub_selector = rule->sub_selector.value_or(""), + .key = report_key, + .attribute = rule->attribute, + }; + select_request.attribute_requests.push_back( + std::move(attribute_request)); + } + select_requests.push_back(std::move(select_request)); + } + + sequenced_task_runner_->PostTaskAndReplyWithResult( + FROM_HERE, + base::BindOnce(&query_element_attributes, html, select_requests), + base::BindOnce(&ContentScraperImpl::OnBrowserParsedElementAttributes, + weak_ptr_factory_.GetWeakPtr(), is_strict_scrape, + std::move(interim_result), std::move(callback))); + } + + private: + void ProcessStandardRule(const std::string& report_key, + const ScrapeRule& rule, + const std::string& root_selector, + const GURL& url, + PageScrapeResult* scrape_result) { + std::string value; + if (rule.attribute == kUrlAttrId) { + value = url.spec(); + } else if (rule.attribute == kCountryCodeAttrId) { + value = server_config_loader_->GetLastServerConfig().location; + } + auto refined_value = ExecuteRefineFunctions(rule.functions_applied, value); + if (!refined_value) { + return; + } + auto& fields = scrape_result->fields[root_selector]; + if (fields.empty()) { + fields.emplace_back(); + } + fields[0].Set(report_key, *refined_value); + } + + void OnRendererScrapedElementAttributes( + bool is_strict_scrape, + std::unique_ptr scrape_result, + PageScrapeResultCallback callback, + std::vector attribute_results) { + const auto* url_details = + server_config_loader_->GetLastPatterns().GetMatchingURLPattern( + scrape_result->url, is_strict_scrape); + if (!url_details) { + return; + } + for (const auto& attribute_result : attribute_results) { + const auto rule_group = + url_details->scrape_rule_groups.find(attribute_result->root_selector); + if (rule_group == url_details->scrape_rule_groups.end()) { + continue; + } + base::Value::Dict attribute_values; + for (const auto& [key, value_str] : attribute_result->attribute_values) { + ProcessAttributeValue(rule_group->second, *scrape_result, key, + value_str, attribute_values); + } + scrape_result->fields[attribute_result->root_selector].push_back( + std::move(attribute_values)); + } + std::move(callback).Run(std::move(scrape_result)); + } + + void OnBrowserParsedElementAttributes( + bool is_strict_scrape, + std::unique_ptr scrape_result, + PageScrapeResultCallback callback, + rust::Vec attribute_results) { + const auto* url_details = + server_config_loader_->GetLastPatterns().GetMatchingURLPattern( + scrape_result->url, is_strict_scrape); + if (!url_details) { + return; + } + for (const auto& attribute_result : attribute_results) { + const auto root_selector = std::string(attribute_result.root_selector); + const auto rule_group = + url_details->scrape_rule_groups.find(root_selector); + if (rule_group == url_details->scrape_rule_groups.end()) { + continue; + } + base::Value::Dict attribute_values; + for (const auto& pair : attribute_result.attribute_pairs) { + ProcessAttributeValue( + rule_group->second, *scrape_result, std::string(pair.key), + pair.value.empty() ? std::nullopt + : std::make_optional(pair.value), + attribute_values); + } + scrape_result->fields[root_selector].push_back( + std::move(attribute_values)); + } + std::move(callback).Run(std::move(scrape_result)); + } + + std::optional ExecuteRefineFunctions( + const RefineFunctionList& function_list, + std::string value) { + std::optional result = value; + for (const auto& function_args : function_list) { + if (function_args.empty()) { + continue; + } + const auto& func_name = function_args[0]; + if (func_name == kRefineSplitFuncId) { + if (function_args.size() >= 3 && function_args[1].is_string() && + function_args[2].is_int()) { + result = RefineSplit(*result, function_args[1].GetString(), + function_args[2].GetInt()); + } + } else if (func_name == kRefineMaskURLFuncId) { + result = MaskURL(GURL(value)); + } else if (func_name == kRefineParseURLFuncId) { + if (function_args.size() >= 3 && function_args[1].is_string() && + function_args[2].is_string()) { + result = RefineParseURL(*result, function_args[1].GetString(), + function_args[2].GetString()); + } + } else if (func_name == kRefineJsonExtractFuncId) { + if (function_args.size() >= 3 && function_args[1].is_string() && + function_args[2].is_bool()) { + result = RefineJsonExtract(*result, function_args[1].GetString(), + function_args[2].GetBool()); + } + } + if (!result) { + break; + } + } + return result; + } + + void ProcessAttributeValue(const ScrapeRuleGroup& rule_group, + PageScrapeResult& scrape_result, + std::string key, + std::optional value_str, + base::Value::Dict& attribute_values) { + const auto rule = rule_group.find(key); + if (rule == rule_group.end()) { + return; + } + base::Value value; + if (value_str) { + value_str = + ExecuteRefineFunctions(rule->second->functions_applied, *value_str); + + if (value_str) { + if (rule->second->rule_type == ScrapeRuleType::kSearchQuery || + rule->second->rule_type == ScrapeRuleType::kWidgetTitle) { + scrape_result.query = value_str; + } + + value = base::Value(*value_str); + } + } + attribute_values.Set(key, std::move(value)); + } + + scoped_refptr sequenced_task_runner_; + + raw_ptr server_config_loader_; + + base::WeakPtrFactory weak_ptr_factory_{this}; +}; + } // namespace +std::unique_ptr ContentScraper::Create( + const ServerConfigLoader* server_config_loader) { + return std::make_unique(server_config_loader); +} + PageScrapeResult::PageScrapeResult(GURL url, std::string id) : url(url), id(id) {} PageScrapeResult::~PageScrapeResult() = default; -ContentScraper::ContentScraper(const ServerConfigLoader* server_config_loader) - : sequenced_task_runner_(base::ThreadPool::CreateSequencedTaskRunner( - {base::TaskPriority::BEST_EFFORT, - base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN})), - server_config_loader_(server_config_loader) {} - -ContentScraper::~ContentScraper() = default; - base::Value PageScrapeResult::SerializeToValue() { base::Value::Dict result; base::Value::Dict fields_dict; @@ -151,232 +399,5 @@ std::unique_ptr PageScrapeResult::FromValue( return result; } -void ContentScraper::ScrapePage(const GURL& url, - bool is_strict_scrape, - mojom::DocumentExtractor* document_extractor, - PageScrapeResultCallback callback) { - const auto* url_details = - server_config_loader_->GetLastPatterns().GetMatchingURLPattern( - url, is_strict_scrape); - if (!url_details) { - return; - } - auto interim_result = - std::make_unique(url, url_details->id); - - std::vector select_requests; - for (const auto& [selector, group] : url_details->scrape_rule_groups) { - auto select_request = mojom::SelectRequest::New(); - select_request->root_selector = selector; - for (const auto& [report_key, rule] : group) { - if (rule->rule_type == ScrapeRuleType::kStandard) { - ProcessStandardRule(report_key, *rule, selector, url, - interim_result.get()); - continue; - } - auto attribute_request = mojom::SelectAttributeRequest::New(); - attribute_request->sub_selector = rule->sub_selector; - attribute_request->attribute = rule->attribute; - attribute_request->key = report_key; - - select_request->attribute_requests.push_back( - std::move(attribute_request)); - } - select_requests.push_back(std::move(select_request)); - } - - document_extractor->QueryElementAttributes( - std::move(select_requests), - base::BindOnce(&ContentScraper::OnScrapedElementAttributes, - base::Unretained(this), is_strict_scrape, - std::move(interim_result), std::move(callback))); -} - -void ContentScraper::ParseAndScrapePage( - const GURL& url, - bool is_strict_scrape, - std::unique_ptr prev_result, - std::string html, - PageScrapeResultCallback callback) { - const auto* url_details = - server_config_loader_->GetLastPatterns().GetMatchingURLPattern( - url, is_strict_scrape); - if (!url_details) { - return; - } - auto interim_result = std::move(prev_result); - - std::vector select_requests; - for (const auto& [selector, group] : url_details->scrape_rule_groups) { - rust_document_extractor::SelectRequest select_request; - select_request.root_selector = selector; - for (const auto& [report_key, rule] : group) { - if (rule->rule_type == ScrapeRuleType::kStandard) { - ProcessStandardRule(report_key, *rule, selector, url, - interim_result.get()); - continue; - } - rust_document_extractor::SelectAttributeRequest attribute_request{ - .sub_selector = rule->sub_selector.value_or(""), - .key = report_key, - .attribute = rule->attribute, - }; - select_request.attribute_requests.push_back(std::move(attribute_request)); - } - select_requests.push_back(std::move(select_request)); - } - - sequenced_task_runner_->PostTaskAndReplyWithResult( - FROM_HERE, - base::BindOnce(&rust_document_extractor::query_element_attributes, html, - select_requests), - base::BindOnce(&ContentScraper::OnRustElementAttributes, - weak_ptr_factory_.GetWeakPtr(), is_strict_scrape, - std::move(interim_result), std::move(callback))); -} - -void ContentScraper::ProcessStandardRule(const std::string& report_key, - const ScrapeRule& rule, - const std::string& root_selector, - const GURL& url, - PageScrapeResult* scrape_result) { - std::string value; - if (rule.attribute == kUrlAttrId) { - value = url.spec(); - } else if (rule.attribute == kCountryCodeAttrId) { - value = server_config_loader_->GetLastServerConfig().location; - } - auto refined_value = ExecuteRefineFunctions(rule.functions_applied, value); - if (!refined_value) { - return; - } - auto& fields = scrape_result->fields[root_selector]; - if (fields.empty()) { - fields.emplace_back(); - } - fields[0].Set(report_key, *refined_value); -} - -void ContentScraper::OnScrapedElementAttributes( - bool is_strict_scrape, - std::unique_ptr scrape_result, - PageScrapeResultCallback callback, - std::vector attribute_results) { - const auto* url_details = - server_config_loader_->GetLastPatterns().GetMatchingURLPattern( - scrape_result->url, is_strict_scrape); - if (!url_details) { - return; - } - for (const auto& attribute_result : attribute_results) { - const auto rule_group = - url_details->scrape_rule_groups.find(attribute_result->root_selector); - if (rule_group == url_details->scrape_rule_groups.end()) { - continue; - } - base::Value::Dict attribute_values; - for (const auto& [key, value_str] : attribute_result->attribute_values) { - ProcessAttributeValue(rule_group->second, *scrape_result, key, value_str, - attribute_values); - } - scrape_result->fields[attribute_result->root_selector].push_back( - std::move(attribute_values)); - } - std::move(callback).Run(std::move(scrape_result)); -} - -void ContentScraper::OnRustElementAttributes( - bool is_strict_scrape, - std::unique_ptr scrape_result, - PageScrapeResultCallback callback, - rust::Vec attribute_results) { - const auto* url_details = - server_config_loader_->GetLastPatterns().GetMatchingURLPattern( - scrape_result->url, is_strict_scrape); - if (!url_details) { - return; - } - for (const auto& attribute_result : attribute_results) { - const auto root_selector = std::string(attribute_result.root_selector); - const auto rule_group = url_details->scrape_rule_groups.find(root_selector); - if (rule_group == url_details->scrape_rule_groups.end()) { - continue; - } - base::Value::Dict attribute_values; - for (const auto& pair : attribute_result.attribute_pairs) { - ProcessAttributeValue( - rule_group->second, *scrape_result, std::string(pair.key), - pair.value.empty() ? std::nullopt - : std::make_optional(pair.value), - attribute_values); - } - scrape_result->fields[root_selector].push_back(std::move(attribute_values)); - } - std::move(callback).Run(std::move(scrape_result)); -} - -std::optional ContentScraper::ExecuteRefineFunctions( - const RefineFunctionList& function_list, - std::string value) { - std::optional result = value; - for (const auto& function_args : function_list) { - if (function_args.empty()) { - continue; - } - const auto& func_name = function_args[0]; - if (func_name == kRefineSplitFuncId) { - if (function_args.size() >= 3 && function_args[1].is_string() && - function_args[2].is_int()) { - result = RefineSplit(*result, function_args[1].GetString(), - function_args[2].GetInt()); - } - } else if (func_name == kRefineMaskURLFuncId) { - result = MaskURL(GURL(value)); - } else if (func_name == kRefineParseURLFuncId) { - if (function_args.size() >= 3 && function_args[1].is_string() && - function_args[2].is_string()) { - result = RefineParseURL(*result, function_args[1].GetString(), - function_args[2].GetString()); - } - } else if (func_name == kRefineJsonExtractFuncId) { - if (function_args.size() >= 3 && function_args[1].is_string() && - function_args[2].is_bool()) { - result = RefineJsonExtract(*result, function_args[1].GetString(), - function_args[2].GetBool()); - } - } - if (!result) { - break; - } - } - return result; -} - -void ContentScraper::ProcessAttributeValue( - const ScrapeRuleGroup& rule_group, - PageScrapeResult& scrape_result, - std::string key, - std::optional value_str, - base::Value::Dict& attribute_values) { - const auto rule = rule_group.find(key); - if (rule == rule_group.end()) { - return; - } - base::Value value; - if (value_str) { - value_str = - ExecuteRefineFunctions(rule->second->functions_applied, *value_str); - - if (value_str) { - if (rule->second->rule_type == ScrapeRuleType::kSearchQuery || - rule->second->rule_type == ScrapeRuleType::kWidgetTitle) { - scrape_result.query = value_str; - } - - value = base::Value(*value_str); - } - } - attribute_values.Set(key, std::move(value)); -} } // namespace web_discovery diff --git a/components/web_discovery/browser/content_scraper.h b/components/web_discovery/browser/content_scraper.h index 779f2587ec6..ec0b199bbb5 100644 --- a/components/web_discovery/browser/content_scraper.h +++ b/components/web_discovery/browser/content_scraper.h @@ -13,8 +13,6 @@ #include "base/containers/flat_map.h" #include "base/functional/callback.h" #include "base/values.h" -#include "brave/components/web_discovery/browser/document_extractor/lib.rs.h" -#include "brave/components/web_discovery/browser/patterns.h" #include "brave/components/web_discovery/browser/server_config_loader.h" #include "brave/components/web_discovery/common/web_discovery.mojom.h" #include "url/gurl.h" @@ -60,55 +58,22 @@ class ContentScraper { using PageScrapeResultCallback = base::OnceCallback)>; - explicit ContentScraper(const ServerConfigLoader* server_config_loader); - ~ContentScraper(); + static std::unique_ptr Create( + const ServerConfigLoader* server_config_loader); - ContentScraper(const ContentScraper&) = delete; - ContentScraper& operator=(const ContentScraper&) = delete; + virtual ~ContentScraper() = default; // For initial page scrape in renderer - void ScrapePage(const GURL& url, - bool is_strict_scrape, - mojom::DocumentExtractor* document_extractor, - PageScrapeResultCallback callback); - // For subsequent double fetches after initial scrape - void ParseAndScrapePage(const GURL& url, + virtual void ScrapePage(const GURL& url, bool is_strict_scrape, - std::unique_ptr prev_result, - std::string html, - PageScrapeResultCallback callback); - - private: - void ProcessStandardRule(const std::string& report_key, - const ScrapeRule& rule, - const std::string& root_selector, - const GURL& url, - PageScrapeResult* scrape_result); - void OnScrapedElementAttributes( - bool is_strict_scrape, - std::unique_ptr scrape_result, - PageScrapeResultCallback callback, - std::vector attribute_results); - void OnRustElementAttributes( - bool is_strict_scrape, - std::unique_ptr scrape_result, - PageScrapeResultCallback callback, - rust::Vec attribute_results); - - std::optional ExecuteRefineFunctions( - const RefineFunctionList& function_list, - std::string value); - void ProcessAttributeValue(const ScrapeRuleGroup& rule_group, - PageScrapeResult& scrape_result, - std::string key, - std::optional value_str, - base::Value::Dict& attribute_values); - - scoped_refptr sequenced_task_runner_; - - raw_ptr server_config_loader_; - - base::WeakPtrFactory weak_ptr_factory_{this}; + mojom::DocumentExtractor* document_extractor, + PageScrapeResultCallback callback) = 0; + // For subsequent double fetches after initial scrape + virtual void ParseAndScrapePage(const GURL& url, + bool is_strict_scrape, + std::unique_ptr prev_result, + std::string html, + PageScrapeResultCallback callback) = 0; }; } // namespace web_discovery diff --git a/components/web_discovery/browser/document_extractor/.gitignore b/components/web_discovery/browser/document_extractor/.gitignore index 436cbd991cd..d9510d9f679 100644 --- a/components/web_discovery/browser/document_extractor/.gitignore +++ b/components/web_discovery/browser/document_extractor/.gitignore @@ -1,2 +1,3 @@ target example/main +Cargo.lock diff --git a/components/web_discovery/browser/document_extractor/Cargo.lock b/components/web_discovery/browser/document_extractor/Cargo.lock deleted file mode 100644 index dd4a2a1e741..00000000000 --- a/components/web_discovery/browser/document_extractor/Cargo.lock +++ /dev/null @@ -1,767 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "autocfg" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "cc" -version = "1.0.98" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "convert_case" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" - -[[package]] -name = "cssparser" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a" -dependencies = [ - "cssparser-macros", - "dtoa-short", - "itoa", - "matches", - "phf", - "proc-macro2", - "quote", - "smallvec", - "syn 1.0.109", -] - -[[package]] -name = "cssparser-macros" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" -dependencies = [ - "quote", - "syn 2.0.66", -] - -[[package]] -name = "cxx" -version = "1.0.122" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb497fad022245b29c2a0351df572e2d67c1046bcef2260ebc022aec81efea82" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.122" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688c799a4a846f1c0acb9f36bb9c6272d9b3d9457f3633c7753c6057270df13c" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.122" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928bc249a7e3cd554fd2e8e08a426e9670c50bbfc9a621653cfa9accc9641783" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "derive_more" -version = "0.99.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" -dependencies = [ - "convert_case", - "proc-macro2", - "quote", - "rustc_version", - "syn 1.0.109", -] - -[[package]] -name = "document-extractor-cxx" -version = "0.1.0" -dependencies = [ - "cxx", - "html5ever", - "kuchikiki", -] - -[[package]] -name = "dtoa" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" - -[[package]] -name = "dtoa-short" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbaceec3c6e4211c79e7b1800fb9680527106beb2f9c51904a3210c03a448c74" -dependencies = [ - "dtoa", -] - -[[package]] -name = "futf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" -dependencies = [ - "mac", - "new_debug_unreachable", -] - -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - -[[package]] -name = "getrandom" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.11.0+wasi-snapshot-preview1", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "html5ever" -version = "0.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5c13fb08e5d4dfc151ee5e88bae63f7773d61852f3bdc73c9f4b9e1bde03148" -dependencies = [ - "log", - "mac", - "markup5ever", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown", -] - -[[package]] -name = "itoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - -[[package]] -name = "kuchikiki" -version = "0.8.4-speedreader" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af921794306a6885fc7510012ce12015d2cf0bfdba82fb217b9c2caf324a4618" -dependencies = [ - "cssparser", - "html5ever", - "indexmap", - "matches", - "selectors", -] - -[[package]] -name = "libc" -version = "0.2.155" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" - -[[package]] -name = "link-cplusplus" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9" -dependencies = [ - "cc", -] - -[[package]] -name = "lock_api" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" - -[[package]] -name = "mac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" - -[[package]] -name = "markup5ever" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd" -dependencies = [ - "log", - "phf", - "phf_codegen", - "string_cache", - "string_cache_codegen", - "tendril", -] - -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - -[[package]] -name = "new_debug_unreachable" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" - -[[package]] -name = "nodrop" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" - -[[package]] -name = "once_cell" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" - -[[package]] -name = "parking_lot" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets", -] - -[[package]] -name = "phf" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" -dependencies = [ - "phf_macros", - "phf_shared 0.8.0", - "proc-macro-hack", -] - -[[package]] -name = "phf_codegen" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" -dependencies = [ - "phf_generator 0.8.0", - "phf_shared 0.8.0", -] - -[[package]] -name = "phf_generator" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" -dependencies = [ - "phf_shared 0.8.0", - "rand 0.7.3", -] - -[[package]] -name = "phf_generator" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" -dependencies = [ - "phf_shared 0.10.0", - "rand 0.8.5", -] - -[[package]] -name = "phf_macros" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c" -dependencies = [ - "phf_generator 0.8.0", - "phf_shared 0.8.0", - "proc-macro-hack", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "phf_shared" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" -dependencies = [ - "siphasher", -] - -[[package]] -name = "phf_shared" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" -dependencies = [ - "siphasher", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "precomputed-hash" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" - -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - -[[package]] -name = "proc-macro2" -version = "1.0.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", - "rand_pcg", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.15", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", -] - -[[package]] -name = "rand_pcg" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" -dependencies = [ - "rand_core 0.5.1", -] - -[[package]] -name = "redox_syscall" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" -dependencies = [ - "bitflags 2.5.0", -] - -[[package]] -name = "rustc_version" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" -dependencies = [ - "semver", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "selectors" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe" -dependencies = [ - "bitflags 1.3.2", - "cssparser", - "derive_more", - "fxhash", - "log", - "matches", - "phf", - "phf_codegen", - "precomputed-hash", - "servo_arc", - "smallvec", - "thin-slice", -] - -[[package]] -name = "semver" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" - -[[package]] -name = "serde" -version = "1.0.203" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.203" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "servo_arc" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432" -dependencies = [ - "nodrop", - "stable_deref_trait", -] - -[[package]] -name = "siphasher" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" - -[[package]] -name = "smallvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "string_cache" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" -dependencies = [ - "new_debug_unreachable", - "once_cell", - "parking_lot", - "phf_shared 0.10.0", - "precomputed-hash", - "serde", -] - -[[package]] -name = "string_cache_codegen" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" -dependencies = [ - "phf_generator 0.10.0", - "phf_shared 0.10.0", - "proc-macro2", - "quote", -] - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.66" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "tendril" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" -dependencies = [ - "futf", - "mac", - "utf-8", -] - -[[package]] -name = "thin-slice" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" - -[[package]] -name = "unicode-ident" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "windows-targets" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" diff --git a/components/web_discovery/browser/document_extractor/lib.rs b/components/web_discovery/browser/document_extractor/lib.rs index 4db51614a39..b742340e256 100644 --- a/components/web_discovery/browser/document_extractor/lib.rs +++ b/components/web_discovery/browser/document_extractor/lib.rs @@ -13,7 +13,7 @@ use kuchikiki::{ traits::TendrilSink, }; -#[cxx::bridge(namespace = "rust_document_extractor")] +#[cxx::bridge(namespace = "web_discovery")] mod ffi { pub struct SelectAttributeRequest { /// An optional selector for an element within the current selected element. diff --git a/components/web_discovery/browser/web_discovery_service.cc b/components/web_discovery/browser/web_discovery_service.cc index be83948593f..eda322b1768 100644 --- a/components/web_discovery/browser/web_discovery_service.cc +++ b/components/web_discovery/browser/web_discovery_service.cc @@ -120,8 +120,7 @@ void WebDiscoveryService::OnConfigChange() { void WebDiscoveryService::OnPatternsLoaded() { if (!content_scraper_) { - content_scraper_ = - std::make_unique(server_config_loader_.get()); + content_scraper_ = ContentScraper::Create(server_config_loader_.get()); } if (!double_fetcher_) { double_fetcher_ = std::make_unique(