diff --git a/components/speedreader/rust/lib/src/readability/src/dom.rs b/components/speedreader/rust/lib/src/readability/src/dom.rs index c300f0624d7..809e7e7bbde 100644 --- a/components/speedreader/rust/lib/src/readability/src/dom.rs +++ b/components/speedreader/rust/lib/src/readability/src/dom.rs @@ -1,7 +1,9 @@ -use markup5ever_rcdom::NodeData::{Element, Text}; -use markup5ever_rcdom::{Handle, Node}; use html5ever::tendril::StrTendril; -use html5ever::{Attribute, QualName, LocalName}; +use html5ever::tendril::TendrilSink; +use html5ever::{parse_document, ParseOpts}; +use html5ever::{Attribute, LocalName, QualName}; +use markup5ever_rcdom::NodeData::{Comment, Element, Text}; +use markup5ever_rcdom::{Handle, Node, RcDom}; use std::rc::Rc; use std::str::FromStr; @@ -177,7 +179,7 @@ pub fn has_nodes(handle: &Handle, tag_names: &[&'static LocalName]) -> bool { return true; } } - + if match child.data { Element { .. } => has_nodes(child, tag_names), _ => false, @@ -203,3 +205,53 @@ pub fn text_children_count(handle: &Handle) -> usize { } count } + +pub fn previous_element_sibling<'a>( + handle: &Handle, + siblings: &'a Vec, +) -> Option<&'a Handle> { + let mut prev: Option<&Handle> = None; + for child in siblings.iter() { + if Rc::ptr_eq(handle, child) { + break; + } + if let Element { .. } = child.data { + prev = Some(child); + } + } + prev +} + +pub fn parse_inner(contents: StrTendril) -> Option { + let dom = parse_document(RcDom::default(), ParseOpts::default()).one(contents); + let document = dom.document.clone(); + let html = document.children.borrow().get(0)?.clone(); + let body = html.children.borrow().get(1)?.clone(); + let img = body.children.borrow().get(0)?.clone(); + Some(img) +} + +pub fn is_single_image(handle: &Handle) -> bool { + match handle.data { + Element { ref name, .. } => { + if name.local == local_name!("img") { + return true; + } + } + Text { ref contents } => { + if !contents.borrow().trim().is_empty() { + return false; + } + } + Comment { .. } => (), + _ => { + return false; + } + } + + let children = handle.children.borrow(); + if children.len() != 1 { + return false; + } + return is_single_image(&children[0]); +} diff --git a/components/speedreader/rust/lib/src/readability/src/extractor.rs b/components/speedreader/rust/lib/src/readability/src/extractor.rs index d9fbaf70081..56dbb2c0f14 100644 --- a/components/speedreader/rust/lib/src/readability/src/extractor.rs +++ b/components/speedreader/rust/lib/src/readability/src/extractor.rs @@ -36,7 +36,7 @@ where extract_dom(&mut dom, url, &HashMap::new()) } -pub fn preprocess(input: &mut R) -> Result +pub fn preprocess(input: &mut R) -> Result where R: Read, { @@ -47,7 +47,14 @@ where let mut title = Title::default(); let handle = dom.document.clone(); scorer::preprocess(&mut dom, handle, &mut title); - Ok(title) + let mut bytes = vec![]; + let document: SerializableHandle = dom.document.clone().into(); + serialize(&mut bytes, &document, serialize::SerializeOpts::default())?; + let content = String::from_utf8(bytes).unwrap_or_default(); + Ok(Product { + title: title.title, + content: content, + }) } pub fn extract_dom( @@ -154,6 +161,14 @@ mod tests { use super::*; use std::io::Cursor; + fn normalize_output(input: &str) -> String { + return input + .lines() + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + .collect(); + } + #[test] fn test_extract_title() { let data = r#" @@ -170,7 +185,7 @@ mod tests { } #[test] - fn test_prefer_meta() { + fn test_title_prefer_meta() { let data = r#" @@ -185,4 +200,77 @@ mod tests { "Raspberry Pi 3 - All-time bestselling computer in UK" ); } + + #[test] + fn unwrap_noscript_img_simple() { + let input = r#" + + + + "#; + let expected = r#" + + + + + + "#; + let mut cursor = Cursor::new(input); + let product = preprocess(&mut cursor).unwrap(); + assert_eq!( + normalize_output(expected), + normalize_output(&product.content) + ); + } + + #[test] + fn unwrap_noscript_img_delete_preceding() { + let input = r#" + + + + "#; + let expected = r#" + + + + + "#; + let mut cursor = Cursor::new(input); + let product = preprocess(&mut cursor).unwrap(); + assert_eq!( + normalize_output(expected), + normalize_output(&product.content) + ); + } + + #[test] + fn unwrap_noscript_img_nested() { + let input = r#" + + + + + "#; + let expected = r#" + + + + + + "#; + + let mut cursor = Cursor::new(input); + let product = preprocess(&mut cursor).unwrap(); + assert_eq!( + normalize_output(expected), + normalize_output(&product.content) + ); + } } diff --git a/components/speedreader/rust/lib/src/readability/src/scorer.rs b/components/speedreader/rust/lib/src/readability/src/scorer.rs index e642f35677e..a3473b38e09 100644 --- a/components/speedreader/rust/lib/src/readability/src/scorer.rs +++ b/components/speedreader/rust/lib/src/readability/src/scorer.rs @@ -1,4 +1,5 @@ use dom; +use html5ever::tendril::StrTendril; use html5ever::tree_builder::TreeSink; use html5ever::tree_builder::{ElementFlags, NodeOrText}; use html5ever::{LocalName, QualName}; @@ -200,6 +201,48 @@ pub fn get_metadata(handle: &Handle, title: &mut Title) { } } +fn get_inner_img(dom: &mut RcDom, handle: &Handle) -> Option { + let children = handle.children.borrow(); + let child = children.get(0)?; + if let Text { ref contents } = child.data { + let s: StrTendril = contents.borrow().clone(); + let inner = dom::parse_inner(s)?; + if dom::is_single_image(&inner) { + if let Element { + ref name, + ref attrs, + .. + } = inner.data + { + let img = dom.create_element( + name.clone(), + attrs.borrow().to_vec(), + ElementFlags::default(), + ); + return Some(img); + } + } + } + None +} + +fn unwrap_noscript( + dom: &mut RcDom, + handle: &Handle, + parent: &Handle, + useless_nodes: &mut Vec, + new_children: &mut Vec, +) { + if let Some(img) = get_inner_img(dom, handle) { + new_children.push(img); + if let Some(prev) = dom::previous_element_sibling(handle, &parent.children.borrow()) { + if dom::is_single_image(prev) { + useless_nodes.push(prev.clone()); + } + } + } +} + pub fn preprocess(mut dom: &mut RcDom, handle: Handle, mut title: &mut Title) -> bool { if let Element { ref name, @@ -208,16 +251,19 @@ pub fn preprocess(mut dom: &mut RcDom, handle: Handle, mut title: &mut Title) -> } = handle.data { match name.local { - local_name!("script") | local_name!("link") | local_name!("style") => return true, + local_name!("script") + | local_name!("noscript") + | local_name!("link") + | local_name!("style") => return true, local_name!("title") => { if !title.is_meta && title.title.is_empty() { dom::extract_text(&handle, &mut title.title, true); title.is_meta = false; } - }, + } local_name!("meta") => { get_metadata(&handle, title); - }, + } _ => (), } for attr_name in ["id", "class", "itemProp"].iter() { @@ -232,6 +278,7 @@ pub fn preprocess(mut dom: &mut RcDom, handle: Handle, mut title: &mut Title) -> } } let mut useless_nodes = vec![]; + let mut new_children = vec![]; let mut paragraph_nodes = vec![]; let mut br_count = 0; for child in handle.children.borrow().iter() { @@ -239,10 +286,21 @@ pub fn preprocess(mut dom: &mut RcDom, handle: Handle, mut title: &mut Title) -> useless_nodes.push(child.clone()); } match child.data { - Element { ref name, .. } => match name.local { - local_name!("br") => br_count += 1, - _ => br_count = 0, - }, + Element { ref name, .. } => { + match name.local { + local_name!("br") => br_count += 1, + _ => br_count = 0, + } + if name.local == local_name!("noscript") { + unwrap_noscript( + &mut dom, + &child, + &handle, + &mut useless_nodes, + &mut new_children, + ); + } + } Text { ref contents } => { let s = contents.borrow(); if br_count >= 2 && !s.trim().is_empty() { @@ -253,6 +311,9 @@ pub fn preprocess(mut dom: &mut RcDom, handle: Handle, mut title: &mut Title) -> _ => (), } } + for node in new_children.iter() { + dom.append(&handle, NodeOrText::AppendNode(node.clone())); + } for node in useless_nodes.iter() { dom.remove_from_parent(node); }