speedreader: Unwrap images from <noscript> blocks

Some sites, such as Medium and BBC, put <img> tags as text inside of a
<noscript> tag. They use this to achieve a blurred image effect, where
the original blurred image is replaced with the higher resolution image
after some Javascript on the page runs.

In reader mode we aren't interested in that functionality. For all
<noscript> tags, we parse the text and check if there is a single image
element embedded inside of it. If there is, we pull that into our DOM
during the preprocess step and replace the <noscript> with the <img>. If
the <noscript> tag had an <img> element as the preceding sibling we
remove that image to avoid rendering both the blurred and final image.

components/speedreader/rust/lib/src/readability/src/dom.rs

@@ -1,7 +1,9 @@
-use markup5ever_rcdom::NodeData::{Element, Text};
-use markup5ever_rcdom::{Handle, Node};
 use html5ever::tendril::StrTendril;
-use html5ever::{Attribute, QualName, LocalName};
+use html5ever::tendril::TendrilSink;
+use html5ever::{parse_document, ParseOpts};
+use html5ever::{Attribute, LocalName, QualName};
+use markup5ever_rcdom::NodeData::{Comment, Element, Text};
+use markup5ever_rcdom::{Handle, Node, RcDom};
 use std::rc::Rc;
 use std::str::FromStr;
 
@@ -177,7 +179,7 @@ pub fn has_nodes(handle: &Handle, tag_names: &[&'static LocalName]) -> bool {
                 return true;
             }
         }
-        
+
         if match child.data {
             Element { .. } => has_nodes(child, tag_names),
             _ => false,
@@ -203,3 +205,53 @@ pub fn text_children_count(handle: &Handle) -> usize {
     }
     count
 }
+
+pub fn previous_element_sibling<'a>(
+    handle: &Handle,
+    siblings: &'a Vec<Handle>,
+) -> Option<&'a Handle> {
+    let mut prev: Option<&Handle> = None;
+    for child in siblings.iter() {
+        if Rc::ptr_eq(handle, child) {
+            break;
+        }
+        if let Element { .. } = child.data {
+            prev = Some(child);
+        }
+    }
+    prev
+}
+
+pub fn parse_inner(contents: StrTendril) -> Option<Handle> {
+    let dom = parse_document(RcDom::default(), ParseOpts::default()).one(contents);
+    let document = dom.document.clone();
+    let html = document.children.borrow().get(0)?.clone();
+    let body = html.children.borrow().get(1)?.clone();
+    let img = body.children.borrow().get(0)?.clone();
+    Some(img)
+}
+
+pub fn is_single_image(handle: &Handle) -> bool {
+    match handle.data {
+        Element { ref name, .. } => {
+            if name.local == local_name!("img") {
+                return true;
+            }
+        }
+        Text { ref contents } => {
+            if !contents.borrow().trim().is_empty() {
+                return false;
+            }
+        }
+        Comment { .. } => (),
+        _ => {
+            return false;
+        }
+    }
+
+    let children = handle.children.borrow();
+    if children.len() != 1 {
+        return false;
+    }
+    return is_single_image(&children[0]);
+}

components/speedreader/rust/lib/src/readability/src/extractor.rs

@@ -36,7 +36,7 @@ where
     extract_dom(&mut dom, url, &HashMap::new())
 }
 
-pub fn preprocess<R>(input: &mut R) -> Result<Title, std::io::Error>
+pub fn preprocess<R>(input: &mut R) -> Result<Product, std::io::Error>
 where
     R: Read,
 {
@@ -47,7 +47,14 @@ where
     let mut title = Title::default();
     let handle = dom.document.clone();
     scorer::preprocess(&mut dom, handle, &mut title);
-    Ok(title)
+    let mut bytes = vec![];
+    let document: SerializableHandle = dom.document.clone().into();
+    serialize(&mut bytes, &document, serialize::SerializeOpts::default())?;
+    let content = String::from_utf8(bytes).unwrap_or_default();
+    Ok(Product {
+        title: title.title,
+        content: content,
+    })
 }
 
 pub fn extract_dom<S: ::std::hash::BuildHasher>(
@@ -154,6 +161,14 @@ mod tests {
     use super::*;
     use std::io::Cursor;
 
+    fn normalize_output(input: &str) -> String {
+        return input
+            .lines()
+            .map(|line| line.trim())
+            .filter(|line| !line.is_empty())
+            .collect();
+    }
+
     #[test]
     fn test_extract_title() {
         let data = r#"
@@ -170,7 +185,7 @@ mod tests {
     }
 
     #[test]
-    fn test_prefer_meta() {
+    fn test_title_prefer_meta() {
         let data = r#"
         <head>
         <meta property="og:title" content="Raspberry Pi 3 - All-time bestselling computer in UK"/>
@@ -185,4 +200,77 @@ mod tests {
             "Raspberry Pi 3 - All-time bestselling computer in UK"
         );
     }
+
+    #[test]
+    fn unwrap_noscript_img_simple() {
+        let input = r#"
+        <body>
+          <noscript>
+            <img src="https://example.com/image.png">
+          </noscript>
+        </body>
+        "#;
+        let expected = r#"
+        <html><head></head>
+        <body>
+          <img src="https://example.com/image.png">
+        </body>
+        </html>
+        "#;
+        let mut cursor = Cursor::new(input);
+        let product = preprocess(&mut cursor).unwrap();
+        assert_eq!(
+            normalize_output(expected),
+            normalize_output(&product.content)
+        );
+    }
+
+    #[test]
+    fn unwrap_noscript_img_delete_preceding() {
+        let input = r#"
+        <body>
+          <img src="https://example.com/image.png">
+          <noscript>
+            <img src="https://example.com/image.png">
+          </noscript>
+        </body>"#;
+        let expected = r#"
+        <html><head></head>
+        <body>
+          <img src="https://example.com/image.png">
+        </body>
+        </html>"#;
+        let mut cursor = Cursor::new(input);
+        let product = preprocess(&mut cursor).unwrap();
+        assert_eq!(
+            normalize_output(expected),
+            normalize_output(&product.content)
+        );
+    }
+
+    #[test]
+    fn unwrap_noscript_img_nested() {
+        let input = r#"
+        <body>
+          <img src="https://example.com/image.png">
+          <noscript>
+            <span><img src="https://example.com/image.png"></span>
+          </noscript>
+        </body>
+        "#;
+        let expected = r#"
+        <html><head></head>
+        <body>
+          <img src="https://example.com/image.png">
+        </body>
+        </html>
+        "#;
+
+        let mut cursor = Cursor::new(input);
+        let product = preprocess(&mut cursor).unwrap();
+        assert_eq!(
+            normalize_output(expected),
+            normalize_output(&product.content)
+        );
+    }
 }

components/speedreader/rust/lib/src/readability/src/scorer.rs

@@ -1,4 +1,5 @@
 use dom;
+use html5ever::tendril::StrTendril;
 use html5ever::tree_builder::TreeSink;
 use html5ever::tree_builder::{ElementFlags, NodeOrText};
 use html5ever::{LocalName, QualName};
@@ -200,6 +201,48 @@ pub fn get_metadata(handle: &Handle, title: &mut Title) {
     }
 }
 
+fn get_inner_img(dom: &mut RcDom, handle: &Handle) -> Option<Handle> {
+    let children = handle.children.borrow();
+    let child = children.get(0)?;
+    if let Text { ref contents } = child.data {
+        let s: StrTendril = contents.borrow().clone();
+        let inner = dom::parse_inner(s)?;
+        if dom::is_single_image(&inner) {
+            if let Element {
+                ref name,
+                ref attrs,
+                ..
+            } = inner.data
+            {
+                let img = dom.create_element(
+                    name.clone(),
+                    attrs.borrow().to_vec(),
+                    ElementFlags::default(),
+                );
+                return Some(img);
+            }
+        }
+    }
+    None
+}
+
+fn unwrap_noscript(
+    dom: &mut RcDom,
+    handle: &Handle,
+    parent: &Handle,
+    useless_nodes: &mut Vec<Handle>,
+    new_children: &mut Vec<Handle>,
+) {
+    if let Some(img) = get_inner_img(dom, handle) {
+        new_children.push(img);
+        if let Some(prev) = dom::previous_element_sibling(handle, &parent.children.borrow()) {
+            if dom::is_single_image(prev) {
+                useless_nodes.push(prev.clone());
+            }
+        }
+    }
+}
+
 pub fn preprocess(mut dom: &mut RcDom, handle: Handle, mut title: &mut Title) -> bool {
     if let Element {
         ref name,
@@ -208,16 +251,19 @@ pub fn preprocess(mut dom: &mut RcDom, handle: Handle, mut title: &mut Title) ->
     } = handle.data
     {
         match name.local {
-            local_name!("script") | local_name!("link") | local_name!("style") => return true,
+            local_name!("script")
+            | local_name!("noscript")
+            | local_name!("link")
+            | local_name!("style") => return true,
             local_name!("title") => {
                 if !title.is_meta && title.title.is_empty() {
                     dom::extract_text(&handle, &mut title.title, true);
                     title.is_meta = false;
                 }
-            },
+            }
             local_name!("meta") => {
                 get_metadata(&handle, title);
-            },
+            }
             _ => (),
         }
         for attr_name in ["id", "class", "itemProp"].iter() {
@@ -232,6 +278,7 @@ pub fn preprocess(mut dom: &mut RcDom, handle: Handle, mut title: &mut Title) ->
         }
     }
     let mut useless_nodes = vec![];
+    let mut new_children = vec![];
     let mut paragraph_nodes = vec![];
     let mut br_count = 0;
     for child in handle.children.borrow().iter() {
@@ -239,10 +286,21 @@ pub fn preprocess(mut dom: &mut RcDom, handle: Handle, mut title: &mut Title) ->
             useless_nodes.push(child.clone());
         }
         match child.data {
-            Element { ref name, .. } => match name.local {
-                local_name!("br") => br_count += 1,
-                _ => br_count = 0,
-            },
+            Element { ref name, .. } => {
+                match name.local {
+                    local_name!("br") => br_count += 1,
+                    _ => br_count = 0,
+                }
+                if name.local == local_name!("noscript") {
+                    unwrap_noscript(
+                        &mut dom,
+                        &child,
+                        &handle,
+                        &mut useless_nodes,
+                        &mut new_children,
+                    );
+                }
+            }
             Text { ref contents } => {
                 let s = contents.borrow();
                 if br_count >= 2 && !s.trim().is_empty() {
@@ -253,6 +311,9 @@ pub fn preprocess(mut dom: &mut RcDom, handle: Handle, mut title: &mut Title) ->
             _ => (),
         }
     }
+    for node in new_children.iter() {
+        dom.append(&handle, NodeOrText::AppendNode(node.clone()));
+    }
     for node in useless_nodes.iter() {
         dom.remove_from_parent(node);
     }