Add feature + params for backup results service (#36339)

This commit is contained in:
Darnell Andries
2026-05-12 13:56:24 -07:00
committed by GitHub
parent a92c7a0fd1
commit 5ab1e39cd4
8 changed files with 406 additions and 6 deletions
+1
View File
@@ -24,6 +24,7 @@ source_set("brave_search") {
"//chrome/browser/content_settings:content_settings_factory",
"//chrome/browser/profiles",
"//components/keyed_service/core",
"//components/prefs",
"//content/public/browser",
"//services/network/public/cpp",
]
@@ -5,25 +5,38 @@
#include "brave/components/brave_search/browser/backup_results_service.h"
#include "base/base64.h"
#include "base/containers/map_util.h"
#include "base/test/bind.h"
#include "base/test/scoped_feature_list.h"
#include "base/time/time.h"
#include "brave/browser/brave_search/backup_results_service_factory.h"
#include "brave/components/brave_search/browser/prefs.h"
#include "brave/components/brave_search/common/features.h"
#include "chrome/browser/browser_process.h"
#include "chrome/browser/profiles/profile.h"
#include "chrome/browser/ui/browser.h"
#include "chrome/test/base/in_process_browser_test.h"
#include "components/prefs/pref_service.h"
#include "content/public/test/browser_test.h"
#include "content/public/test/browser_test_utils.h"
#include "content/public/test/content_mock_cert_verifier.h"
#include "net/dns/mock_host_resolver.h"
#include "net/http/http_request_headers.h"
#include "net/test/embedded_test_server/embedded_test_server.h"
#include "net/test/embedded_test_server/http_request.h"
#include "net/test/embedded_test_server/http_response.h"
#include "third_party/abseil-cpp/absl/strings/str_format.h"
#include "third_party/blink/public/common/user_agent/user_agent_metadata.h"
namespace brave_search {
namespace {
constexpr char kTestCustomHeaderName[] = "X-Custom-Header";
constexpr char kTestCustomHeaderValue[] = "test-value";
constexpr char kTestUAOverride[] = "TestBrowser/1.0";
constexpr char kTestInitPath[] = "/test";
constexpr char kTestInitHtml[] = R"(
<!doctype html>
@@ -86,6 +99,13 @@ class BackupResultsServiceBrowserTest : public InProcessBrowserTest {
response->set_content_type("text/html");
auto url = request.GetURL();
if (auto* v = base::FindOrNull(request.headers, kTestCustomHeaderName)) {
last_custom_header_ = *v;
}
if (auto* v = base::FindOrNull(request.headers,
net::HttpRequestHeaders::kUserAgent)) {
last_user_agent_ = *v;
}
if (url.path() == kTestInitPath) {
response->set_content(redirect_to_invalid_domain_
? kTestInitInvalidRedirectHtml
@@ -117,6 +137,9 @@ class BackupResultsServiceBrowserTest : public InProcessBrowserTest {
std::unique_ptr<net::EmbeddedTestServer> https_server_;
raw_ptr<BackupResultsService> backup_results_service_;
std::optional<std::string> last_custom_header_;
std::optional<std::string> last_user_agent_;
};
IN_PROC_BROWSER_TEST_F(BackupResultsServiceBrowserTest, BasicRenderAndLoad) {
@@ -132,6 +155,9 @@ IN_PROC_BROWSER_TEST_F(BackupResultsServiceBrowserTest, BasicRenderAndLoad) {
EXPECT_EQ(kTestFinalHtml, result->html);
EXPECT_EQ(net::HTTP_OK, result->final_status_code);
}
EXPECT_FALSE(last_custom_header_);
EXPECT_TRUE(last_user_agent_);
EXPECT_NE(last_user_agent_, kTestUAOverride);
run_loop.Quit();
}));
@@ -186,6 +212,9 @@ IN_PROC_BROWSER_TEST_F(BackupResultsServiceBrowserTest, CookieHeader) {
EXPECT_EQ(kTestFinalHtml, result->html);
EXPECT_EQ(net::HTTP_OK, result->final_status_code);
}
EXPECT_FALSE(last_custom_header_);
EXPECT_TRUE(last_user_agent_);
EXPECT_NE(last_user_agent_, kTestUAOverride);
run_loop.Quit();
}));
@@ -220,4 +249,229 @@ IN_PROC_BROWSER_TEST_F(BackupResultsServiceFullRenderBrowserTest, FullRender) {
run_loop.Run();
}
class BackupResultsServiceDisabledBrowserTest
: public BackupResultsServiceBrowserTest {
public:
BackupResultsServiceDisabledBrowserTest() {
scoped_feature_list_.InitAndDisableFeature(features::kBackupResults);
}
};
IN_PROC_BROWSER_TEST_F(BackupResultsServiceDisabledBrowserTest,
FeatureDisabled) {
base::RunLoop run_loop;
GURL url = https_server_->GetURL("google.ca", kTestInitPath);
backup_results_service_->FetchBackupResults(
url, std::nullopt,
base::BindLambdaForTesting(
[&](std::optional<BackupResultsService::BackupResults> result) {
EXPECT_FALSE(result.has_value());
run_loop.Quit();
}));
run_loop.Run();
}
class BackupResultsServiceFeatureHeadersBrowserTest
: public BackupResultsServiceBrowserTest {
public:
BackupResultsServiceFeatureHeadersBrowserTest() {
scoped_feature_list_.InitWithFeaturesAndParameters(
{{features::kBackupResults,
{{"headers", absl::StrFormat("{\"%s\":\"%s\"}", kTestCustomHeaderName,
kTestCustomHeaderValue)}}}},
{});
}
};
IN_PROC_BROWSER_TEST_F(BackupResultsServiceFeatureHeadersBrowserTest,
SimpleURLLoader) {
base::RunLoop run_loop;
GURL url = https_server_->GetURL("google.co.uk", kTestFinalPath);
net::HttpRequestHeaders headers;
headers.SetHeader(net::HttpRequestHeaders::kCookie, "testcookie=value");
backup_results_service_->FetchBackupResults(
url, headers,
base::BindLambdaForTesting(
[&](std::optional<BackupResultsService::BackupResults> result) {
EXPECT_TRUE(result.has_value());
EXPECT_EQ(last_custom_header_, kTestCustomHeaderValue);
EXPECT_TRUE(last_user_agent_);
EXPECT_NE(last_user_agent_, kTestUAOverride);
run_loop.Quit();
}));
run_loop.Run();
}
IN_PROC_BROWSER_TEST_F(BackupResultsServiceFeatureHeadersBrowserTest,
WebContents) {
base::RunLoop run_loop;
GURL url = https_server_->GetURL("google.ca", kTestInitPath);
backup_results_service_->FetchBackupResults(
url, std::nullopt,
base::BindLambdaForTesting(
[&](std::optional<BackupResultsService::BackupResults> result) {
EXPECT_TRUE(result.has_value());
EXPECT_EQ(last_custom_header_, kTestCustomHeaderValue);
EXPECT_TRUE(last_user_agent_);
EXPECT_NE(last_user_agent_, kTestUAOverride);
run_loop.Quit();
}));
run_loop.Run();
}
class BackupResultsServiceUAOverrideBrowserTest
: public BackupResultsServiceBrowserTest {
public:
BackupResultsServiceUAOverrideBrowserTest() {
scoped_feature_list_.InitWithFeaturesAndParameters(
{{features::kBackupResults, {{"ua_override", kTestUAOverride}}}}, {});
}
};
IN_PROC_BROWSER_TEST_F(BackupResultsServiceUAOverrideBrowserTest, WebContents) {
base::RunLoop run_loop;
GURL url = https_server_->GetURL("google.ca", kTestInitPath);
backup_results_service_->FetchBackupResults(
url, std::nullopt,
base::BindLambdaForTesting(
[&](std::optional<BackupResultsService::BackupResults> result) {
EXPECT_TRUE(result.has_value());
EXPECT_EQ(last_user_agent_, kTestUAOverride);
run_loop.Quit();
}));
run_loop.Run();
}
class BackupResultsServiceUAOverrideWithMetadataBrowserTest
: public BackupResultsServiceBrowserTest {
public:
BackupResultsServiceUAOverrideWithMetadataBrowserTest() {
blink::UserAgentMetadata ua_metadata;
ua_metadata.brand_version_list = {{"TestBrowser", "1"}};
ua_metadata.brand_full_version_list = {{"TestBrowser", "1.0"}};
ua_metadata.full_version = "1.0";
ua_metadata.platform = "Linux";
ua_metadata.platform_version = "1.0";
ua_metadata.architecture = "x86";
ua_metadata.model = "";
ua_metadata.mobile = false;
auto marshalled = blink::UserAgentMetadata::Marshal(ua_metadata);
std::string encoded =
marshalled ? base::Base64Encode(*marshalled) : std::string();
scoped_feature_list_.InitWithFeaturesAndParameters(
{{features::kBackupResults,
{{"ua_override", kTestUAOverride}, {"ua_metadata", encoded}}}},
{});
}
};
IN_PROC_BROWSER_TEST_F(BackupResultsServiceUAOverrideWithMetadataBrowserTest,
WebContents) {
base::RunLoop run_loop;
GURL url = https_server_->GetURL("google.ca", kTestInitPath);
backup_results_service_->FetchBackupResults(
url, std::nullopt,
base::BindLambdaForTesting(
[&](std::optional<BackupResultsService::BackupResults> result) {
EXPECT_TRUE(result.has_value());
EXPECT_EQ(last_user_agent_, kTestUAOverride);
run_loop.Quit();
}));
run_loop.Run();
}
class BackupResultsServiceDailyLimitBrowserTest
: public BackupResultsServiceBrowserTest {
public:
BackupResultsServiceDailyLimitBrowserTest() {
scoped_feature_list_.InitWithFeaturesAndParameters(
{{features::kBackupResults, {{"max_daily_requests", "2"}}}}, {});
}
};
// Verifies that once the daily limit is reached, subsequent fetches fail
// immediately without hitting the network.
IN_PROC_BROWSER_TEST_F(BackupResultsServiceDailyLimitBrowserTest,
DailyLimitEnforced) {
GURL url = https_server_->GetURL("google.co.uk", kTestFinalPath);
net::HttpRequestHeaders headers;
headers.SetHeader(net::HttpRequestHeaders::kCookie, "testcookie=value");
// First two requests should succeed (limit is 2).
for (int i = 0; i < 2; i++) {
base::RunLoop run_loop;
backup_results_service_->FetchBackupResults(
url, headers,
base::BindLambdaForTesting(
[&](std::optional<BackupResultsService::BackupResults> result) {
EXPECT_TRUE(result.has_value());
run_loop.Quit();
}));
run_loop.Run();
}
// Third request should be rejected immediately.
{
base::RunLoop run_loop;
backup_results_service_->FetchBackupResults(
url, headers,
base::BindLambdaForTesting(
[&](std::optional<BackupResultsService::BackupResults> result) {
EXPECT_FALSE(result.has_value());
run_loop.Quit();
}));
run_loop.Run();
}
// Simulate a day passing by backdating the window start pref.
g_browser_process->local_state()->SetTime(
prefs::kBackupResultsDailyRequestWindowStart,
base::Time::Now() - base::Days(1) - base::Seconds(1));
// First request of the new window should succeed.
{
base::RunLoop run_loop;
backup_results_service_->FetchBackupResults(
url, headers,
base::BindLambdaForTesting(
[&](std::optional<BackupResultsService::BackupResults> result) {
EXPECT_TRUE(result.has_value());
run_loop.Quit();
}));
run_loop.Run();
}
}
// Verifies that the default param value (-1) does not impose any daily limit.
IN_PROC_BROWSER_TEST_F(BackupResultsServiceBrowserTest, NoDailyLimitByDefault) {
GURL url = https_server_->GetURL("google.co.uk", kTestFinalPath);
net::HttpRequestHeaders headers;
headers.SetHeader(net::HttpRequestHeaders::kCookie, "testcookie=value");
for (int i = 0; i < 5; i++) {
base::RunLoop run_loop;
backup_results_service_->FetchBackupResults(
url, headers,
base::BindLambdaForTesting(
[&](std::optional<BackupResultsService::BackupResults> result) {
EXPECT_TRUE(result.has_value());
run_loop.Quit();
}));
run_loop.Run();
}
}
} // namespace brave_search
@@ -8,17 +8,24 @@
#include <algorithm>
#include <utility>
#include "base/base64.h"
#include "base/byte_count.h"
#include "base/feature_list.h"
#include "base/functional/bind.h"
#include "base/json/json_reader.h"
#include "base/rand_util.h"
#include "base/time/time.h"
#include "base/values.h"
#include "brave/components/brave_search/browser/backup_results_allowed_urls.h"
#include "brave/components/brave_search/browser/backup_results_service.h"
#include "brave/components/brave_search/browser/prefs.h"
#include "brave/components/brave_search/common/features.h"
#include "brave/components/brave_shields/core/browser/brave_shields_utils.h"
#include "chrome/browser/browser_process.h"
#include "chrome/browser/content_extraction/inner_html.h"
#include "chrome/browser/content_settings/host_content_settings_map_factory.h"
#include "chrome/browser/profiles/profile.h"
#include "components/prefs/pref_service.h"
#include "content/public/browser/browser_thread.h"
#include "content/public/browser/navigation_controller.h"
#include "content/public/browser/navigation_handle.h"
@@ -32,6 +39,7 @@
#include "services/network/public/cpp/simple_url_loader.h"
#include "services/network/public/mojom/url_response_head.mojom.h"
#include "third_party/blink/public/common/navigation/navigation_policy.h"
#include "third_party/blink/public/common/user_agent/user_agent_metadata.h"
#include "third_party/blink/public/common/web_preferences/web_preferences.h"
#if BUILDFLAG(IS_ANDROID)
@@ -113,7 +121,8 @@ WEB_CONTENTS_USER_DATA_KEY_IMPL(BackupResultsWebContentsObserver);
BackupResultsServiceImpl::BackupResultsServiceImpl(Profile* profile)
: profile_(profile),
backup_results_metrics_(g_browser_process->local_state()) {
local_state_(g_browser_process->local_state()),
backup_results_metrics_(local_state_) {
profile_->AddObserver(this);
}
BackupResultsServiceImpl::~BackupResultsServiceImpl() = default;
@@ -122,7 +131,8 @@ void BackupResultsServiceImpl::FetchBackupResults(
const GURL& url,
std::optional<net::HttpRequestHeaders> headers,
BackupResultsCallback callback) {
if (!profile_) {
if (!profile_ || !base::FeatureList::IsEnabled(features::kBackupResults) ||
UpdateDailyRequestCount()) {
std::move(callback).Run(std::nullopt);
return;
}
@@ -184,9 +194,11 @@ void BackupResultsServiceImpl::FetchBackupResults(
static_cast<blink::NavigationDownloadType>(i);
load_url_params.download_policy.SetDisallowed(type);
}
if (request->headers) {
load_url_params.extra_headers = request->headers->ToString();
auto extra_headers = GetExtraHeaders(request->headers);
if (!extra_headers.IsEmpty()) {
load_url_params.extra_headers = extra_headers.ToString();
}
MaybeApplyUserAgentOverride(*request->web_contents, load_url_params);
if (!request->web_contents->GetController().LoadURLWithParams(
load_url_params)) {
CleanupAndDispatchResult(request, std::nullopt);
@@ -282,6 +294,54 @@ base::WeakPtr<BackupResultsService> BackupResultsServiceImpl::GetWeakPtr() {
return weak_ptr_factory_.GetWeakPtr();
}
net::HttpRequestHeaders BackupResultsServiceImpl::GetExtraHeaders(
const std::optional<net::HttpRequestHeaders>& request_headers) {
if (!feature_headers_) {
feature_headers_.emplace();
const std::string& headers_json = features::kBackupResultsHeaders.Get();
if (!headers_json.empty()) {
auto parsed = base::JSONReader::Read(headers_json, base::JSON_PARSE_RFC);
if (parsed && parsed->is_dict()) {
for (const auto [name, value] : parsed->GetDict()) {
if (value.is_string()) {
feature_headers_->SetHeader(name, value.GetString());
}
}
}
}
}
net::HttpRequestHeaders extra_headers = *feature_headers_;
if (request_headers) {
extra_headers.MergeFrom(*request_headers);
}
return extra_headers;
}
void BackupResultsServiceImpl::MaybeApplyUserAgentOverride(
content::WebContents& web_contents,
content::NavigationController::LoadURLParams& load_url_params) {
if (!ua_override_) {
const std::string& ua_string = features::kBackupResultsUAOverride.Get();
if (ua_string.empty()) {
return;
}
ua_override_.emplace();
ua_override_->ua_string_override = ua_string;
const std::string& encoded_ua = features::kBackupResultsUAMetadata.Get();
if (!encoded_ua.empty()) {
std::string decoded;
if (base::Base64Decode(encoded_ua, &decoded)) {
ua_override_->ua_metadata_override =
blink::UserAgentMetadata::Demarshal(decoded);
}
}
}
web_contents.SetUserAgentOverride(*ua_override_,
/*override_in_new_tabs=*/true);
load_url_params.override_user_agent =
content::NavigationController::UA_OVERRIDE_TRUE;
}
void BackupResultsServiceImpl::MakeSimpleURLLoaderRequest(
PendingRequestList::iterator pending_request,
const GURL& url) {
@@ -302,8 +362,11 @@ void BackupResultsServiceImpl::MakeSimpleURLLoaderRequest(
resource_request->site_for_cookies = net::SiteForCookies::FromUrl(url);
}
if (pending_request->headers) {
resource_request->headers = *pending_request->headers;
resource_request->headers = GetExtraHeaders(pending_request->headers);
if (ua_override_) {
resource_request->headers.SetHeader(net::HttpRequestHeaders::kUserAgent,
ua_override_->ua_string_override);
}
pending_request->simple_url_loader = network::SimpleURLLoader::Create(
@@ -359,6 +422,32 @@ void BackupResultsServiceImpl::CleanupAndDispatchResult(
}
}
bool BackupResultsServiceImpl::UpdateDailyRequestCount() {
const int limit = features::kBackupResultsMaxDailyRequests.Get();
if (limit == -1) {
return false;
}
const auto now = base::Time::Now();
const auto window_start =
local_state_->GetTime(prefs::kBackupResultsDailyRequestWindowStart);
if (window_start.is_null() || (now - window_start) >= base::Days(1)) {
local_state_->SetTime(prefs::kBackupResultsDailyRequestWindowStart, now);
local_state_->SetInteger(prefs::kBackupResultsDailyRequestCount, 1);
return false;
}
const int count =
local_state_->GetInteger(prefs::kBackupResultsDailyRequestCount);
if (count >= limit) {
return true;
}
local_state_->SetInteger(prefs::kBackupResultsDailyRequestCount, count + 1);
return false;
}
void BackupResultsServiceImpl::OnProfileWillBeDestroyed(Profile* profile) {
Shutdown();
}
@@ -17,9 +17,12 @@
#include "brave/components/brave_search/browser/backup_results_metrics.h"
#include "brave/components/brave_search/browser/backup_results_service.h"
#include "chrome/browser/profiles/profile_observer.h"
#include "content/public/browser/navigation_controller.h"
#include "net/http/http_request_headers.h"
#include "third_party/blink/public/common/user_agent/user_agent_metadata.h"
#include "url/gurl.h"
class PrefService;
class Profile;
namespace content {
@@ -100,7 +103,22 @@ class BackupResultsServiceImpl : public BackupResultsService,
void CleanupAndDispatchResult(PendingRequestList::iterator pending_request,
std::optional<BackupResults> result);
void MaybeApplyUserAgentOverride(
content::WebContents& web_contents,
content::NavigationController::LoadURLParams& load_url_params);
net::HttpRequestHeaders GetExtraHeaders(
const std::optional<net::HttpRequestHeaders>& request_headers);
// Returns true if the daily request limit has been reached, false otherwise.
bool UpdateDailyRequestCount();
raw_ptr<Profile> profile_;
raw_ptr<PrefService> local_state_;
// Cached on first use; nullopt if param is absent/invalid.
std::optional<blink::UserAgentOverride> ua_override_;
std::optional<net::HttpRequestHeaders> feature_headers_;
PendingRequestList pending_requests_;
@@ -34,6 +34,8 @@ BackupResultsMetrics::~BackupResultsMetrics() = default;
void BackupResultsMetrics::RegisterPrefs(PrefRegistrySimple* registry) {
registry->RegisterTimePref(prefs::kBackupResultsLastQueryTime, {});
registry->RegisterListPref(prefs::kBackupResultsFailuresStorage);
registry->RegisterIntegerPref(prefs::kBackupResultsDailyRequestCount, 0);
registry->RegisterTimePref(prefs::kBackupResultsDailyRequestWindowStart, {});
}
void BackupResultsMetrics::RecordQuery(bool is_failure) {
+4
View File
@@ -22,6 +22,10 @@ inline constexpr char kBackupResultsLastQueryTime[] =
"brave.search.backup_results.last_query_time";
inline constexpr char kBackupResultsFailuresStorage[] =
"brave.search.backup_results.failures_storage";
inline constexpr char kBackupResultsDailyRequestCount[] =
"brave.search.backup_results.daily_request_count";
inline constexpr char kBackupResultsDailyRequestWindowStart[] =
"brave.search.backup_results.daily_request_window_start";
} // namespace prefs
} // namespace brave_search
@@ -5,6 +5,8 @@
#include "brave/components/brave_search/common/features.h"
#include <string>
#include "base/feature_list.h"
namespace brave_search::features {
@@ -30,4 +32,20 @@ bool IsBackupResultsFullRenderEnabled() {
return base::FeatureList::IsEnabled(kBackupResultsFullRender);
}
BASE_FEATURE(kBackupResults,
"BraveSearchBackupResults",
base::FEATURE_ENABLED_BY_DEFAULT);
const base::FeatureParam<std::string> kBackupResultsHeaders{&kBackupResults,
"headers", ""};
const base::FeatureParam<std::string> kBackupResultsUAOverride{
&kBackupResults, "ua_override", ""};
const base::FeatureParam<std::string> kBackupResultsUAMetadata{
&kBackupResults, "ua_metadata", ""};
const base::FeatureParam<int> kBackupResultsMaxDailyRequests{
&kBackupResults, "max_daily_requests", -1};
} // namespace brave_search::features
+14
View File
@@ -31,6 +31,20 @@ extern const base::FeatureParam<int> kBackupResultsFullRenderMaxRequests;
bool IsBackupResultsFullRenderEnabled();
// Provides request configuration for the backup results service.
BASE_DECLARE_FEATURE(kBackupResults);
// JSON-serialized headers object {"Header-Name": "value"} to include in
// SimpleURLLoader requests.
extern const base::FeatureParam<std::string> kBackupResultsHeaders;
// UA string override for WebContents and User-Agent header in SimpleURLLoader
// requests.
extern const base::FeatureParam<std::string> kBackupResultsUAOverride;
// Base64-encoded pickled blink::UserAgentMetadata. Only used if
// kBackupResultsUAOverride is also provided.
extern const base::FeatureParam<std::string> kBackupResultsUAMetadata;
// Maximum number of backup results fetches allowed per day. -1 means no limit.
extern const base::FeatureParam<int> kBackupResultsMaxDailyRequests;
} // namespace features
} // namespace brave_search