From 490ec0ffb145dedccc317225b3080b3fe260701a Mon Sep 17 00:00:00 2001 From: Terry Mancey Date: Mon, 2 Feb 2026 20:02:34 +0000 Subject: [PATCH] Implement SERP URL classifier (#33472) This change adds logic to classify navigated URLs as search engine results pages and determine the corresponding search engine bucket (Brave, Google, or Other). The classifier operates solely on URL structure and does not extract or persist query terms, providing a reliable and privacy-safe foundation for SERP detection and metrics collection. This is part of https://github.com/brave/brave-browser/issues/52104. Co-authored-by: Aleksei Seren --- components/serp_metrics/BUILD.gn | 15 +- components/serp_metrics/DEPS | 1 + components/serp_metrics/serp_classifier.cc | 119 ++++++++ components/serp_metrics/serp_classifier.h | 47 ++++ .../serp_metrics/serp_classifier_unittest.cc | 260 ++++++++++++++++++ 5 files changed, 439 insertions(+), 3 deletions(-) create mode 100644 components/serp_metrics/serp_classifier.cc create mode 100644 components/serp_metrics/serp_classifier.h create mode 100644 components/serp_metrics/serp_classifier_unittest.cc diff --git a/components/serp_metrics/BUILD.gn b/components/serp_metrics/BUILD.gn index 2336510775e..8b0439f1cdd 100644 --- a/components/serp_metrics/BUILD.gn +++ b/components/serp_metrics/BUILD.gn @@ -4,9 +4,15 @@ # You can obtain one at https://mozilla.org/MPL/2.0/. static_library("serp_metrics") { - public = [ "serp_metrics.h" ] + public = [ + "serp_classifier.h", + "serp_metrics.h", + ] - sources = [ "serp_metrics.cc" ] + sources = [ + "serp_classifier.cc", + "serp_metrics.cc", + ] public_deps = [ "//base", @@ -45,7 +51,10 @@ static_library("features") { source_set("unit_tests") { testonly = true - sources = [ "serp_metrics_unittest.cc" ] + sources = [ + "serp_classifier_unittest.cc", + "serp_metrics_unittest.cc", + ] deps = [ ":features", diff --git a/components/serp_metrics/DEPS b/components/serp_metrics/DEPS index 6ca0ca7f4ea..ff2ce37fa36 100644 --- a/components/serp_metrics/DEPS +++ b/components/serp_metrics/DEPS @@ -1,4 +1,5 @@ include_rules = [ + "+components/search_engines", "+components/prefs", "+absl", ] diff --git a/components/serp_metrics/serp_classifier.cc b/components/serp_metrics/serp_classifier.cc new file mode 100644 index 00000000000..f25d267b613 --- /dev/null +++ b/components/serp_metrics/serp_classifier.cc @@ -0,0 +1,119 @@ +/* Copyright (c) 2026 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "brave/components/serp_metrics/serp_classifier.h" + +#include "base/check.h" +#include "base/containers/fixed_flat_set.h" +#include "components/search_engines/search_engine_type.h" +#include "components/search_engines/search_terms_data.h" +#include "components/search_engines/template_url.h" +#include "components/search_engines/template_url_service.h" +#include "url/gurl.h" + +namespace serp_metrics { + +namespace { + +constexpr auto kDisallowList = base::MakeFixedFlatSet( + base::sorted_unique, + { + SEARCH_ENGINE_UNKNOWN, + SEARCH_ENGINE_STARTER_PACK_BOOKMARKS, + SEARCH_ENGINE_STARTER_PACK_HISTORY, + SEARCH_ENGINE_STARTER_PACK_TABS, + SEARCH_ENGINE_STARTER_PACK_GEMINI, + SEARCH_ENGINE_STARTER_PACK_PAGE, + SEARCH_ENGINE_STARTER_PACK_AI_MODE, + }); + +} // namespace + +SerpClassifier::SerpClassifier(TemplateURLService* template_url_service) + : template_url_service_(template_url_service) { + CHECK(template_url_service_); + + template_url_service->Load(); +} + +SerpClassifier::~SerpClassifier() = default; + +bool SerpClassifier::IsSameSearchQuery(const GURL& lhs, const GURL& rhs) const { + return NormalizeUrl(lhs) == NormalizeUrl(rhs); +} + +std::optional SerpClassifier::MaybeClassify(const GURL& url) { + if (!template_url_service_->loaded()) { + return std::nullopt; + } + + const GURL normalized_url = NormalizeUrl(url); + + TemplateURL* template_url = + template_url_service_->GetTemplateURLForHost(normalized_url.GetHost()); + if (!template_url) { + return std::nullopt; + } + + if (template_url->type() != TemplateURL::NORMAL) { + // Ignore non-standard search engines (e.g. extension/omnibox). + return std::nullopt; + } + + const SearchTermsData& search_terms_data = + template_url_service_->search_terms_data(); + + if (!template_url->IsSearchURL(normalized_url, search_terms_data)) { + // Not a search URL. + return std::nullopt; + } + + const SearchEngineType search_engine_type = + template_url->GetEngineType(search_terms_data); + if (kDisallowList.contains(search_engine_type)) { + return std::nullopt; + } + + return search_engine_type; +} + +/////////////////////////////////////////////////////////////////////////////// + +GURL SerpClassifier::NormalizeUrl(const GURL& url) const { + if (!url.is_valid()) { + return url; + } + + // Strip the port. Search engine template search URLs never include explicit + // ports, and test servers use random ones. Google tests handle this via + // `switches::kIgnoreGooglePortNumbers`, but that switch is Google-specific + // and does not apply to other hosts. + GURL::Replacements url_replacements; + url_replacements.ClearPort(); + GURL normalized_url = url.ReplaceComponents(url_replacements); + + TemplateURL* template_url = + template_url_service_->GetTemplateURLForHost(normalized_url.GetHost()); + if (!template_url) { + return normalized_url; + } + + const SearchTermsData& search_terms_data = + template_url_service_->search_terms_data(); + + if (!template_url->IsSearchURL(normalized_url, search_terms_data)) { + return normalized_url; + } + + template_url->KeepSearchTermsInURL(normalized_url, search_terms_data, + /*keep_search_intent_params=*/false, + /*normalize_search_terms=*/true, + /*out_url=*/&normalized_url, + /*out_search_terms=*/nullptr); + + return normalized_url; +} + +} // namespace serp_metrics diff --git a/components/serp_metrics/serp_classifier.h b/components/serp_metrics/serp_classifier.h new file mode 100644 index 00000000000..cfb2e50cadb --- /dev/null +++ b/components/serp_metrics/serp_classifier.h @@ -0,0 +1,47 @@ +/* Copyright (c) 2026 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#ifndef BRAVE_COMPONENTS_SERP_METRICS_SERP_CLASSIFIER_H_ +#define BRAVE_COMPONENTS_SERP_METRICS_SERP_CLASSIFIER_H_ + +#include + +#include "base/memory/raw_ptr.h" +#include "components/search_engines/search_engine_type.h" + +class GURL; +class TemplateURLService; + +namespace serp_metrics { + +// SerpClassifier determines whether a URL is a search engine results page and, +// if so, identifies the corresponding search engine. + +class SerpClassifier final { + public: + explicit SerpClassifier(TemplateURLService* template_url_service); + ~SerpClassifier(); + + SerpClassifier(const SerpClassifier&) = delete; + SerpClassifier& operator=(const SerpClassifier&) = delete; + + // Returns `true` if `lhs` and `rhs` represent the same search results page. + bool IsSameSearchQuery(const GURL& lhs, const GURL& rhs) const; + + // Returns the corresponding search engine type if `url` is a SERP. Returns + // `std::nullopt` if `url` is not a SERP or if the navigation repeats the same + // canonical SERP URL consecutively. This avoids double-counting. + std::optional MaybeClassify(const GURL& url); + + private: + // Normalizes a URL so equivalent search results pages compare equal. + GURL NormalizeUrl(const GURL& url) const; + + const raw_ptr template_url_service_; // Not owned. +}; + +} // namespace serp_metrics + +#endif // BRAVE_COMPONENTS_SERP_METRICS_SERP_CLASSIFIER_H_ diff --git a/components/serp_metrics/serp_classifier_unittest.cc b/components/serp_metrics/serp_classifier_unittest.cc new file mode 100644 index 00000000000..dd55b4b7cf7 --- /dev/null +++ b/components/serp_metrics/serp_classifier_unittest.cc @@ -0,0 +1,260 @@ +/* Copyright (c) 2026 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "brave/components/serp_metrics/serp_classifier.h" + +#include + +#include "base/check.h" +#include "base/test/task_environment.h" +#include "brave/components/search_engines/brave_prepopulated_engines.h" +#include "components/search_engines/search_engine_type.h" +#include "components/search_engines/search_engines_test_environment.h" +#include "components/search_engines/template_url.h" +#include "components/search_engines/template_url_data_util.h" +#include "components/search_engines/template_url_prepopulate_data.h" +#include "components/search_engines/template_url_service.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "url/gurl.h" + +namespace serp_metrics { + +namespace { + +TemplateURL* FindByPrepopulateId(TemplateURLService* template_url_service, + int id) { + CHECK(template_url_service); + + for (TemplateURL* template_url : template_url_service->GetTemplateURLs()) { + if (template_url && template_url->prepopulate_id() == id) { + return template_url; + } + } + + return nullptr; +} + +void PrepopulateTemplateURLService(TemplateURLService* template_url_service) { + CHECK(template_url_service); + + // Add all prepopulated engines from Chromium first. + for (const TemplateURLPrepopulateData::PrepopulatedEngine* + prepopulated_engine : TemplateURLPrepopulateData::kAllEngines) { + CHECK(prepopulated_engine); + + if (FindByPrepopulateId(template_url_service, prepopulated_engine->id)) { + continue; + } + + std::unique_ptr template_url_data = + TemplateURLDataFromPrepopulatedEngine(*prepopulated_engine); + CHECK(template_url_data); + template_url_service->Add( + std::make_unique(*template_url_data)); + } + + // Now add Brave-specific prepopulated engines. + for (const auto& [id, prepopulated_engine] : + TemplateURLPrepopulateData::kBraveEngines) { + CHECK(prepopulated_engine); + + if (FindByPrepopulateId(template_url_service, prepopulated_engine->id)) { + continue; + } + + std::unique_ptr template_url_data = + TemplateURLDataFromPrepopulatedEngine(*prepopulated_engine); + CHECK(template_url_data); + template_url_service->Add( + std::make_unique(*template_url_data)); + } +} + +} // namespace + +class SerpClassifierTest : public testing::Test { + protected: + void SetUp() override { + template_url_service_ = + search_engines_test_environment_.template_url_service(); + ASSERT_TRUE(template_url_service_); + + template_url_service_->Load(); + ASSERT_TRUE(template_url_service_->loaded()); + + PrepopulateTemplateURLService(template_url_service_); + } + + TemplateURLService* template_url_service() { return template_url_service_; } + + private: + base::test::SingleThreadTaskEnvironment task_environment_; + search_engines::SearchEnginesTestEnvironment search_engines_test_environment_; + raw_ptr template_url_service_ = nullptr; +}; + +TEST_F(SerpClassifierTest, IsSameSearchQuery) { + SerpClassifier classifier(template_url_service()); + + EXPECT_TRUE(classifier.IsSameSearchQuery( + GURL(R"(https://www.qwant.com/?q=foobar)"), + GURL(R"(https://www.qwant.com/?q=foobar&t=web)"))); +} + +TEST_F(SerpClassifierTest, IsSameSearchQueryWithDifferentParamOrder) { + SerpClassifier classifier(template_url_service()); + + EXPECT_TRUE(classifier.IsSameSearchQuery( + GURL(R"(https://www.qwant.com/?q=foobar)"), + GURL(R"(https://www.qwant.com/?t=web&q=foobar)"))); +} + +TEST_F(SerpClassifierTest, IsNotSameSearchQuery) { + SerpClassifier classifier(template_url_service()); + + EXPECT_FALSE(classifier.IsSameSearchQuery( + GURL(R"(https://www.qwant.com/?q=foo&t=web)"), + GURL(R"(https://www.qwant.com/?q=bar&t=web")"))); +} + +TEST_F(SerpClassifierTest, IsNotSameSearchQueryWithInvalidUrl) { + SerpClassifier classifier(template_url_service()); + + EXPECT_FALSE(classifier.IsSameSearchQuery( + GURL(R"(https://www.qwant.com/?q=foobar)"), GURL("foobar"))); +} + +TEST_F(SerpClassifierTest, ClassifySearchEngines) { + SerpClassifier classifier(template_url_service()); + + /* Brave */ + EXPECT_EQ(SearchEngineType::SEARCH_ENGINE_BRAVE, + classifier.MaybeClassify(GURL( + R"(https://search.brave.com/search?q=foobar&source=web)"))); + + /* Google */ + EXPECT_EQ( + SearchEngineType::SEARCH_ENGINE_GOOGLE, + classifier.MaybeClassify(GURL( + R"(https://www.google.com/search?q=foobar&sca_esv=76156e36b6817723&sxsrf=ANbL-n6QnD8wxx3-mGlxNR8KIZSDgUYRPA%3A1769199851341&source=hp&ei=69hzafnxEtqqwbkPwuva8Qg&iflsig=AFdpzrgAAAAAaXPm-8WLXmtp5HpWJECsjI8tX_VK1brF&ved=0ahUKEwi5k7KFv6KSAxVaVTABHcK1No4Q4dUDCCA&uact=5&oq=foobar&gs_lp=Egdnd3Mtd2l6IgZmb29iYXIyBRAAGIAEMgUQABiABDIPEAAYgAQYsQMYgwEYChgLMg8QABiABBixAxiDARgKGAsyBRAAGIAEMg8QABiABBixAxiDARgKGAsyBRAAGIAEMgwQABiABBixAxgKGAsyDxAAGIAEGLEDGIMBGAoYCzIQEAAYgAQYsQMYgwEYigUYCkjgBVBEWMYFcAF4AJABAJgBZqAB4AKqAQMzLjG4AQPIAQD4AQGYAgWgAu8CqAIKwgIKEAAYAxjqAhiPAcICChAuGAMY6gIYjwHCAhEQLhiABBixAxjRAxiDARjHAcICCxAAGIAEGLEDGIMBwgIUEC4YgAQYsQMYgwEYxwEYigUYrwHCAggQLhiABBixA8ICCBAAGIAEGLEDwgIOEAAYgAQYsQMYgwEYigXCAgUQLhiABMICCxAuGIAEGNEDGMcBwgIOEC4YgAQYsQMY0QMYxwGYAwTxBfuh5nekEME-kgcDNC4xoAezJLIHAzMuMbgH6wLCBwUwLjQuMcgHC4AIAA&sclient=gws-wiz)"))); + + /* DuckDuckGo */ + EXPECT_EQ( + SearchEngineType::SEARCH_ENGINE_DUCKDUCKGO, + classifier.MaybeClassify(GURL( + R"(https://duckduckgo.com/?ia=web&origin=funnel_home_website&t=h_&q=foobar&chip-select=search)"))); + + /* Qwant */ + EXPECT_EQ(SearchEngineType::SEARCH_ENGINE_QWANT, + classifier.MaybeClassify( + GURL(R"(https://www.qwant.com/?q=foobar&t=web)"))); + + /* Bing */ + EXPECT_EQ( + SearchEngineType::SEARCH_ENGINE_BING, + classifier.MaybeClassify(GURL( + R"(https://www.bing.com/search?q=foobar&form=QBLH&sp=-1&ghc=1&lq=0&pq=fooba&sc=12-5&qs=n&sk=&cvid=227EE41587C74448ACE88DFBD62B5E3F)"))); + + /* Ecosia */ + EXPECT_EQ(SearchEngineType::SEARCH_ENGINE_ECOSIA, + classifier.MaybeClassify(GURL( + R"(https://www.ecosia.org/search?method=index&q=foobar)"))); + + /* Daum */ + EXPECT_EQ( + SearchEngineType::SEARCH_ENGINE_DAUM, + classifier.MaybeClassify(GURL( + R"(https://search.daum.net/search?w=tot&DA=YZR&t__nil_searchbox=btn&q=foobar)"))); + + /* Freespoke */ + EXPECT_EQ(SearchEngineType::SEARCH_ENGINE_FREESPOKE, + classifier.MaybeClassify( + GURL(R"(https://freespoke.com/search/web?q=foobar)"))); + + /* Info.com */ + EXPECT_EQ( + SearchEngineType::SEARCH_ENGINE_INFO_COM, + classifier.MaybeClassify(GURL( + R"(https://www.info.com/serp?q=foobar&segment=info.infous.udog2)"))); + + /* Kagi */ + EXPECT_EQ( + SearchEngineType::SEARCH_ENGINE_KAGI, + classifier.MaybeClassify(GURL(R"(https://kagi.com/search?q=foobar)"))); + + /* Lilo */ + EXPECT_EQ(SearchEngineType::SEARCH_ENGINE_LILO, + classifier.MaybeClassify( + GURL(R"(https://search.lilo.org/?q=foobar&t=web)"))); + + /* Mojeek */ + EXPECT_EQ(SearchEngineType::SEARCH_ENGINE_MOJEEK, + classifier.MaybeClassify( + GURL(R"(https://www.mojeek.com/search?q=foobar&theme=dark)"))); + + /* Naver */ + EXPECT_EQ( + SearchEngineType::SEARCH_ENGINE_NAVER, + classifier.MaybeClassify(GURL( + R"(https://search.naver.com/search.naver?where=nexearch&sm=top_hty&fbm=0&ie=utf8&query=foobar&ackey=c1hehqf3)"))); + + /* Nona */ + EXPECT_EQ(SearchEngineType::SEARCH_ENGINE_NONA, + classifier.MaybeClassify(GURL(R"(https://www.nona.de/?q=foobar)"))); + + /* Seznam */ + EXPECT_EQ( + SearchEngineType::SEARCH_ENGINE_SEZNAM, + classifier.MaybeClassify(GURL( + R"(https://search.seznam.cz/?q=foobar&oq=foobar&aq=-1&ms=1960&ks=12&sourceid=trending-hp&sId=QfoZcwCDZlMUQpj9Wg2d)"))); + + /* 360 Search */ + EXPECT_EQ( + SearchEngineType::SEARCH_ENGINE_360, + classifier.MaybeClassify(GURL( + R"(https://www.so.com/s?q=foobar&src=360sou_newhome&ssid=8ea5e221c71149978377a0785867d4ac&sp=a56&cp=0ef00452b0&nlpv=global_place_c_shyc&fr=360sou_newhome)"))); + + /* Sogou */ + EXPECT_EQ( + SearchEngineType::SEARCH_ENGINE_SOGOU, + classifier.MaybeClassify(GURL( + R"(https://www.sogou.com/web?query=foobar&_asf=www.sogou.com&_ast=&w=01019900&p=40040100&ie=utf8&from=index-nologin&s_from=index&sourceid=9_01_03&sessiontime=1769199046301)"))); + + /* Yep */ + EXPECT_EQ(SearchEngineType::SEARCH_ENGINE_YEP, + classifier.MaybeClassify(GURL(R"(https://yep.com/web?q=foobar)"))); + + /* You.com */ + EXPECT_EQ( + SearchEngineType::SEARCH_ENGINE_YOU, + classifier.MaybeClassify(GURL( + R"(https://you.com/search?q=foobar&fromSearchBar=true&chatMode=default)"))); +} + +TEST_F(SerpClassifierTest, DoNotClassifyDisallowedSearchEngines) { + SerpClassifier classifier(template_url_service()); + + EXPECT_FALSE( + classifier.MaybeClassify(GURL(R"(chrome://bookmarks/?q=foobar)"))); + EXPECT_FALSE(classifier.MaybeClassify(GURL(R"(https://www.google.com/)"))); +} + +TEST_F(SerpClassifierTest, DoNotClassifyNonSearchEngine) { + SerpClassifier classifier(template_url_service()); + + EXPECT_FALSE(classifier.MaybeClassify( + GURL(R"(https://www.perplexity.ai/search/new/foo)"))); + EXPECT_FALSE(classifier.MaybeClassify(GURL(R"(https://brave.com/)"))); + EXPECT_FALSE(classifier.MaybeClassify(GURL(R"(https://bar.com/baz)"))); + EXPECT_FALSE(classifier.MaybeClassify(GURL(R"(https://qux.quux.com/corge)"))); + EXPECT_FALSE( + classifier.MaybeClassify(GURL(R"(https://startpage.com/grault)"))); + EXPECT_FALSE( + classifier.MaybeClassify(GURL(R"(https://uk.search.yahoo.com/garply)"))); + EXPECT_FALSE( + classifier.MaybeClassify(GURL(R"(https://search.yahoo.com/waldo)"))); +} + +} // namespace serp_metrics