Discharge7443/brave-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

speedreader: Expand path hints and loosen scoring

Browse Source 
We can loosen the scoring heuristic back to 2 paragraphs since pages
must first pass the URL checking heuristic. Now we can go back to
handling readable sites with sparsely formatted content.

Resolves https://github.com/brave/brave-browser/issues/15981
This commit is contained in:
Kevin Kuehler
2021-05-19 16:44:20 -07:00
parent a92be23871
commit 7dde8faf69
3 changed files with 24 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/speedreader/rust/lib/src/readability/src/statistics.rs
+3 -3
View File
@@ -61,9 +61,9 @@ static LIKELY_CANDIDATES: &'static [&'static str; 6] =
// Stop calculating paragraph length after this limit is reached
const TEXT_LENGTH_SATURATION: usize = 1000;
// For untagged documents, only consider paragraphs with at least 280
// characters. This roughly corresponds four English sentences.
const PARAGRAPH_LENGTH_THRESHOLD_WEBSITE: usize = 280;
// For untagged documents, only consider paragraphs with at least 140
// characters. This roughly corresponds two English sentences.
const PARAGRAPH_LENGTH_THRESHOLD_WEBSITE: usize = 140;
lazy_static! {
// Upper bound of all moz scores, saturated with 6 paragraphs. This is a
components/speedreader/speedreader_util.cc
+1 -1
View File
@@ -15,7 +15,7 @@ namespace {
// Regex pattern for paths like /blog/, /article/, /post/, hinting the page
// is a blog entry, magazine entry, or news article.
constexpr char kReadablePathSingleComponentHints[] =
"(?i)/(blogs?|news|articles?|posts?|amp)/";
"(?i)/(blogs?|news|story|entry|articles?|posts?|amp)(/|$)";
// Regex pattern for matching URL paths of the form /YYYY/MM/DD/, which is
// extremely common for news websites.
constexpr char kReadablePathMultiComponentHints[] = "/\\d\\d\\d\\d/\\d\\d/";
components/speedreader/speedreader_util_unittest.cc
+20
View File
@@ -26,6 +26,16 @@ TEST(SpeedreaderUtilTest, URLHasHints) {
EXPECT_TRUE(URLReadableHintExtractor::GetInstance()->HasHints(
GURL("https://www.nature.com/articles/d41586-021-01332-0")));
// Has "story" in path
EXPECT_TRUE(URLReadableHintExtractor::GetInstance()->HasHints(
GURL("https://www.architecturaldigest.com/story/"
"new-york-city-approved-floating-pool-east-river")));
// Has "entry" in path
EXPECT_TRUE(URLReadableHintExtractor::GetInstance()->HasHints(
GURL("https://www.huffpost.com/entry/"
"asap-rocky-rihanna-relationship_n_60a53b3ce4b09092480b8249")));
// Ignore case on article
EXPECT_TRUE(URLReadableHintExtractor::GetInstance()->HasHints(
GURL("https://lwn.net/Articles/414618/")));
@@ -34,5 +44,15 @@ TEST(SpeedreaderUtilTest, URLHasHints) {
EXPECT_TRUE(URLReadableHintExtractor::GetInstance()->HasHints(
GURL("https://blog.twitter.com/engineering/en_us/topics/open-source/2021/"
"dropping-cache-didnt-drop-cache.html")));
// "story" is final component (no trailing "/")
EXPECT_TRUE(URLReadableHintExtractor::GetInstance()->HasHints(GURL(
"https://abcnews.go.com/Politics/"
"state-dept-ends-policy-denying-us-citizenship-children/"
"story?id=77743483&cid=clicksource_4380645_5_film_strip_icymi_hed")));
// 'b' follows "story"
EXPECT_FALSE(URLReadableHintExtractor::GetInstance()->HasHints(
GURL("https://fake.com/storyboard")));
}
} // namespace speedreader