Refactor client-side cosmetic filtering for more iteration

2020-02-03 16:40:48 -08:00
parent ee3d741616
commit a59867034e
11 changed files with 2130 additions and 767 deletions
@@ -1,7 +1,7 @@
 use_relative_paths = True

 deps = {
-  "vendor/adblock_rust_ffi": "https://github.com/brave/adblock-rust-ffi.git@optional-css-validation",
+  "vendor/adblock_rust_ffi": "https://github.com/brave/adblock-rust-ffi.git@f0afcc18a8c2365140c8c6ccd772bf084cdd2846",
  "vendor/autoplay-whitelist": "https://github.com/brave/autoplay-whitelist.git@ea527a4d36051daedb34421e129c98eda06cb5d3",
  "vendor/extension-whitelist": "https://github.com/brave/extension-whitelist.git@7843f62e26a23c51336330e220e9d7992680aae9",
  "vendor/hashset-cpp": "https://github.com/brave/hashset-cpp.git@6eab0271d014ff09bd9f38abe1e0c117e13e9aa9",
@@ -147,11 +147,12 @@ export const generateClassIdStylesheet = (tabId: number, classes: string[], ids:
  }
 }

-export const cosmeticFilterRuleExceptions = (tabId: number, exceptions: string[]) => {
+export const cosmeticFilterRuleExceptions = (tabId: number, exceptions: string[], randomizedClassName: string) => {
  return {
    type: types.COSMETIC_FILTER_RULE_EXCEPTIONS,
    tabId,
-    exceptions
+    exceptions,
+    randomizedClassName
  }
 }

@@ -2,54 +2,28 @@
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */

+const uuidv4 = require('uuid/v4')
+
 import shieldsPanelActions from '../actions/shieldsPanelActions'

-const generateCosmeticBlockingStylesheet = (hideSelectors: string[], styleSelectors: any) => {
-  let stylesheet = ''
-  if (hideSelectors.length > 0) {
-    stylesheet += hideSelectors[0]
-    for (const selector of hideSelectors.slice(1)) {
-      stylesheet += ',' + selector
+const informTabOfCosmeticRulesToConsider = (tabId: number, hideRules: string[], customStyleRules: any) => {
+  if (hideRules.length !== 0 || (customStyleRules && customStyleRules !== {})) {
+    const message = {
+      type: 'cosmeticFilterConsiderNewRules',
+      hideRules,
+      customStyleRules
    }
-    stylesheet += '{display:none !important;}\n'
+    const options = {
+      frameId: 0
+    }
+    chrome.tabs.sendMessage(tabId, message, options)
  }
-  for (const selector in styleSelectors) {
-    stylesheet += selector + '{' + styleSelectors[selector] + '\n'
-  }
-
-  return stylesheet
 }

 export const injectClassIdStylesheet = (tabId: number, classes: string[], ids: string[], exceptions: string[]) => {
  chrome.braveShields.hiddenClassIdSelectors(classes, ids, exceptions, (jsonSelectors) => {
    const hideSelectors = JSON.parse(jsonSelectors)
-    /*
-    chrome.tabs.insertCSS(tabId, {
-      code: stylesheet,
-      cssOrigin: 'user',
-      runAt: 'document_start'
-    })
-    */
-  })
-}
-
-export const addSiteCosmeticFilter = async (origin: string, cssfilter: string) => {
-  chrome.storage.local.get('cosmeticFilterList', (storeData = {}) => {
-    let storeList = Object.assign({}, storeData.cosmeticFilterList)
-    if (storeList[origin] === undefined || storeList[origin].length === 0) { // nothing in filter list for origin
-      storeList[origin] = [cssfilter]
-    } else { // add entry
-      storeList[origin].push(cssfilter)
-    }
-    chrome.storage.local.set({ 'cosmeticFilterList': storeList })
-  })
-}
-
-export const removeSiteFilter = (origin: string) => {
-  chrome.storage.local.get('cosmeticFilterList', (storeData = {}) => {
-    let storeList = Object.assign({}, storeData.cosmeticFilterList)
-    delete storeList[origin]
-    chrome.storage.local.set({ 'cosmeticFilterList': storeList })
+    informTabOfCosmeticRulesToConsider(tabId, hideSelectors, null)
  })
 }

@@ -60,14 +34,7 @@ export const applyAdblockCosmeticFilters = (tabId: number, hostname: string) =>
      return
    }

-    const stylesheet = generateCosmeticBlockingStylesheet(resources.hide_selectors, resources.style_selectors)
-    if (stylesheet) {
-      chrome.tabs.insertCSS(tabId, {
-        code: stylesheet,
-        cssOrigin: 'user',
-        runAt: 'document_start'
-      })
-    }
+    informTabOfCosmeticRulesToConsider(tabId, resources.hide_selectors, resources.style_selectors)

    if (resources.injected_script) {
      chrome.tabs.executeScript(tabId, {
@@ -76,10 +43,19 @@ export const applyAdblockCosmeticFilters = (tabId: number, hostname: string) =>
      })
    }

-    shieldsPanelActions.cosmeticFilterRuleExceptions(tabId, resources.exceptions)
+    const randomizedClassName = 'b' + uuidv4().split('-').slice(0, 2).join('')
+
+    chrome.tabs.insertCSS(tabId, {
+      code: `.${randomizedClassName} {display: none !important;}`,
+      cssOrigin: 'user',
+      runAt: 'document_start'
+    })
+
+    shieldsPanelActions.cosmeticFilterRuleExceptions(tabId, resources.exceptions, randomizedClassName)
  })
 }

+// User generated cosmetic filtering below
 export const applyCSSCosmeticFilters = (tabId: number, hostname: string) => {
  chrome.storage.local.get('cosmeticFilterList', (storeData = {}) => {
    if (!storeData.cosmeticFilterList) {
@@ -106,3 +82,23 @@ export const applyCSSCosmeticFilters = (tabId: number, hostname: string) => {
 export const removeAllFilters = () => {
  chrome.storage.local.set({ 'cosmeticFilterList': {} })
 }
+
+export const addSiteCosmeticFilter = async (origin: string, cssfilter: string) => {
+  chrome.storage.local.get('cosmeticFilterList', (storeData = {}) => {
+    let storeList = Object.assign({}, storeData.cosmeticFilterList)
+    if (storeList[origin] === undefined || storeList[origin].length === 0) { // nothing in filter list for origin
+      storeList[origin] = [cssfilter]
+    } else { // add entry
+      storeList[origin].push(cssfilter)
+    }
+    chrome.storage.local.set({ 'cosmeticFilterList': storeList })
+  })
+}
+
+export const removeSiteFilter = (origin: string) => {
+  chrome.storage.local.get('cosmeticFilterList', (storeData = {}) => {
+    let storeList = Object.assign({}, storeData.cosmeticFilterList)
+    delete storeList[origin]
+    chrome.storage.local.set({ 'cosmeticFilterList': storeList })
+  })
+}
@@ -368,7 +368,8 @@ export default function shieldsPanelReducer (
      }
      state = shieldsPanelState.saveCosmeticFilterRuleExceptions(state, action.tabId, action.exceptions)
      chrome.tabs.sendMessage(action.tabId, {
-        type: 'cosmeticFilterGenericExceptions'
+        type: 'cosmeticFilterGenericExceptions',
+        randomizedClassName: action.randomizedClassName
      })
      break
    }
@@ -1,13 +1,3 @@
-// Notify the background script as soon as the content script has loaded.
-// chrome.tabs.insertCSS may sometimes fail to inject CSS in a newly navigated
-// page when using the chrome.webNavigation API.
-// See: https://bugs.chromium.org/p/chromium/issues/detail?id=331654#c15
-// The RenderView should always be ready when the content script begins, so
-// this message is used to trigger CSS insertion instead.
-chrome.runtime.sendMessage({
-  type: 'contentScriptsLoaded'
-})
-
 const unique = require('unique-selector').default
 let target: EventTarget | null

@@ -1,80 +1,304 @@
+// Notify the background script as soon as the content script has loaded.
+// chrome.tabs.insertCSS may sometimes fail to inject CSS in a newly navigated
+// page when using the chrome.webNavigation API.
+// See: https://bugs.chromium.org/p/chromium/issues/detail?id=331654#c15
+// The RenderView should always be ready when the content script begins, so
+// this message is used to trigger CSS insertion instead.
+chrome.runtime.sendMessage({
+  type: 'contentScriptsLoaded'
+})
+
+const parseDomain = require('parse-domain')
+
 const queriedIds = new Set()
 const queriedClasses = new Set()
-const regexWhitespace = /\s/

-const getClassesAndIds = function (addedNodes: Element[]) {
-  const ids = []
-  const classes = []
+let notYetQueriedClasses: string[] = []
+let notYetQueriedIds: string[] = []

-  for (const node of addedNodes) {
-    let nodeId = node.id
-    if (nodeId && nodeId.length !== 0) {
-      nodeId = nodeId.trim()
-      if (!queriedIds.has(nodeId) && nodeId.length !== 0) {
-        ids.push(nodeId)
-        queriedIds.add(nodeId)
-      }
+let randomizedClassName: string | undefined = undefined
+
+const handleMutations = function (mutations: any[]) {
+  for (const aMutation of mutations) {
+    if (aMutation.type !== 'attributes') {
+      continue
    }
-    let nodeClass = node.className
-    if (nodeClass && nodeClass.length !== 0 && !regexWhitespace.test(nodeClass)) {
-      if (!queriedClasses.has(nodeClass)) {
-        classes.push(nodeClass)
-        queriedClasses.add(nodeClass)
-      }
-    } else {
-      let nodeClasses = node.classList
-      if (nodeClasses) {
-        let j = nodeClasses.length
-        while (j--) {
-          const nodeClassJ = nodeClasses[j]
-          if (queriedClasses.has(nodeClassJ) === false) {
-            classes.push(nodeClassJ)
-            queriedClasses.add(nodeClassJ)
+
+    // Since we're filtering for attribute modifications, we can be certain
+    // that the targets are always HTMLElements, and never TextNode.
+    const changedElm = aMutation.target
+    switch (aMutation.attributeName) {
+      case 'class':
+        for (const aClassName of changedElm.classList.values()) {
+          if (queriedClasses.has(aClassName) === false) {
+            notYetQueriedClasses.push(aClassName)
+            queriedClasses.add(aClassName)
          }
        }
+        break
+
+      case 'id':
+        const mutatedId = changedElm.id
+        if (queriedIds.has(mutatedId) === false) {
+          notYetQueriedIds.push(mutatedId)
+          queriedIds.add(mutatedId)
+        }
+        break
+    }
+  }
+
+  // Only let the backend know that we've found new classes and id attributes
+  // if (1) the back end has told us its ready to go
+  // (e.g. randomizedClassName has been set) and we have at least one
+  // new class or id to query for.
+  if (randomizedClassName !== undefined &&
+      (notYetQueriedClasses.length !== 0 || notYetQueriedIds.length !== 0)) {
+    chrome.runtime.sendMessage({
+      type: 'hiddenClassIdSelectors',
+      classes: notYetQueriedClasses,
+      ids: notYetQueriedIds
+    })
+    notYetQueriedClasses = []
+    notYetQueriedIds = []
+  }
+}
+
+const cosmeticObserver = new MutationObserver(handleMutations)
+let observerConfig = {
+  subtree: true,
+  attributeFilter: ['id', 'class']
+}
+cosmeticObserver.observe(document.documentElement, observerConfig)
+
+const _parseDomainCache = Object.create(null)
+const getParsedDomain = (aDomain: any) => {
+  const cacheResult = _parseDomainCache[aDomain]
+  if (cacheResult !== undefined) {
+    return cacheResult
+  }
+
+  const newResult = parseDomain(aDomain)
+  _parseDomainCache[aDomain] = newResult
+  return newResult
+}
+
+const _parsedCurrentDomain = getParsedDomain(window.location.host)
+const isFirstPartyUrl = (url: string): boolean => {
+  if (url.startsWith('/')) {
+    return true
+  }
+
+  const parsedTargetDomain = getParsedDomain(url)
+  if (!parsedTargetDomain) {
+    // If we cannot determine the party-ness of the resource,
+    // consider it first-party.
+    console.debug(`Unable to determine party-ness of "${url}"`)
+    return false
+  }
+
+  return (
+    _parsedCurrentDomain.tld === parsedTargetDomain.tld &&
+    _parsedCurrentDomain.domain === parsedTargetDomain.domain
+  )
+}
+
+interface IsFirstPartyQueryResult {
+  foundFirstPartyResource: boolean,
+  foundThirdPartyResource: boolean,
+  foundKnownThirdPartyAd: boolean
+}
+
+/**
+ * Determine whether a given subtree should be considered as "first party" content.
+ *
+ * Uses the following process in making this determination.
+ *   - If the subtree contains any first party resources, the subtree is first party.
+ *   - If the subtree contains no remote resources, the subtree is first party.
+ *   - Otherwise, its 3rd party.
+ *
+ * Note that any instances of "url(" or escape characters in style attributes are
+ * automatically treated as third-party URLs.  These patterns and special cases
+ * were generated from looking at patterns in ads with resources in the style
+ * attribute.
+ *
+ * Similarly, an empty srcdoc attribute is also considered third party, since many
+ * third party ads clear this attribute in practice.
+ *
+ * Finally, special case some ids we know are used only for third party ads.
+ */
+const isSubTreeFirstParty = (elm: Element, possibleQueryResult?: IsFirstPartyQueryResult): boolean => {
+  let queryResult: IsFirstPartyQueryResult
+  let isTopLevel: boolean
+
+  if (possibleQueryResult) {
+    queryResult = possibleQueryResult
+    isTopLevel = false
+  } else {
+    queryResult = {
+      foundFirstPartyResource: false,
+      foundThirdPartyResource: false,
+      foundKnownThirdPartyAd: false
+    }
+    isTopLevel = true
+  }
+
+  if (elm.getAttribute) {
+    if (elm.hasAttribute('id')) {
+      const elmId = elm.getAttribute('id') as string
+      if (elmId.startsWith('google_ads_iframe_') ||
+          elmId.startsWith('div-gpt-ad') ||
+          elmId.startsWith('adfox_')) {
+        queryResult.foundKnownThirdPartyAd = true
+        return false
+      }
+    }
+
+    if (elm.hasAttribute('src')) {
+      const elmSrc = elm.getAttribute('src') as string
+      const elmSrcIsFirstParty = isFirstPartyUrl(elmSrc)
+      if (elmSrcIsFirstParty === true) {
+        queryResult.foundFirstPartyResource = true
+        return true
+      }
+      queryResult.foundThirdPartyResource = true
+    }
+
+    if (elm.hasAttribute('style')) {
+      const elmStyle = elm.getAttribute('style') as string
+      if (elmStyle.includes('url(') ||
+          elmStyle.includes('//')) {
+        queryResult.foundThirdPartyResource = true
+      }
+    }
+
+    if (elm.hasAttribute('srcdoc')) {
+      const elmSrcDoc = elm.getAttribute('srcdoc') as string
+      if (elmSrcDoc.trim() === '') {
+        queryResult.foundThirdPartyResource = true
      }
    }
  }
-  return { classes, ids }
+
+  if (elm.firstChild) {
+    isSubTreeFirstParty(elm.firstChild as Element, queryResult)
+    if (queryResult.foundKnownThirdPartyAd === true) {
+      return false
+    }
+    if (queryResult.foundFirstPartyResource === true) {
+      return true
+    }
+  }
+
+  if (elm.nextSibling) {
+    isSubTreeFirstParty(elm.nextSibling as Element, queryResult)
+    if (queryResult.foundKnownThirdPartyAd === true) {
+      return false
+    }
+    if (queryResult.foundFirstPartyResource === true) {
+      return true
+    }
+  }
+
+  if (isTopLevel === false) {
+    return (queryResult.foundThirdPartyResource === false)
+  }
+
+  const foundText = (elm as HTMLElement).innerText
+  return (
+    queryResult.foundThirdPartyResource === false &&
+    foundText.trim().length > 0
+  )
 }

-const handleNewNodes = (newNodes: Element[]) => {
-  const { classes, ids } = getClassesAndIds(newNodes)
-  chrome.runtime.sendMessage({
-    type: 'hiddenClassIdSelectors',
-    classes,
-    ids
-  })
+const hideSubtree = (elm: HTMLElement) => {
+  if (elm.classList) {
+    if (randomizedClassName) {
+      elm.classList.add(randomizedClassName)
+    } else {
+      console.error('Random class name was not initialized yet')
+    }
+  } else if (elm.parentNode) {
+    (elm.parentNode as HTMLElement).removeChild(elm)
+  }
 }

-function applyCosmeticFilterMutationObserver () {
-  let targetNode = document.documentElement
-  let observer = new MutationObserver(mutations => {
-    const nodeList: Element[] = []
-    for (const mutation of mutations) {
-      for (let nodeIndex = 0; nodeIndex < mutation.addedNodes.length; nodeIndex++) {
-        nodeList.push(mutation.addedNodes[nodeIndex] as Element)
+const alreadyHiddenThirdPartySubTrees = new WeakSet()
+const allSelectorsSet = new Set()
+const firstRunQueue = new Set()
+const secondRunQueue = new Set()
+const finalRunQueue = new Set()
+const allQueues = [firstRunQueue, secondRunQueue, finalRunQueue]
+const numQueues = allQueues.length
+const pumpIntervalMs = 50
+const maxWorkSize = 50
+let queueIsSleeping = false
+
+const pumpCosmeticFilterQueues = () => {
+  if (queueIsSleeping) {
+    return
+  }
+
+  let didPumpAnything = false
+  for (let queueIndex = 0; queueIndex < numQueues; queueIndex += 1) {
+    const currentQueue = allQueues[queueIndex]
+    const nextQueue = allQueues[queueIndex + 1]
+    if (currentQueue.size === 0) {
+      continue
+    }
+
+    const currentWorkLoad = Array.from(currentQueue.values()).slice(0, maxWorkSize)
+    const comboSelector = currentWorkLoad.join(',')
+    const matchingElms = document.querySelectorAll(comboSelector)
+    for (const aMatchingElm of Array.from(matchingElms)) {
+      if (alreadyHiddenThirdPartySubTrees.has(aMatchingElm)) {
+        continue
+      }
+      const elmSubtreeIsFirstParty = isSubTreeFirstParty(aMatchingElm)
+      if (elmSubtreeIsFirstParty === false) {
+        hideSubtree(aMatchingElm as HTMLElement)
+        alreadyHiddenThirdPartySubTrees.add(aMatchingElm)
      }
    }
-    handleNewNodes(nodeList)
-  })
-  let observerConfig = {
-    childList: true,
-    subtree: true
+
+    for (const aUsedSelector of currentWorkLoad) {
+      currentQueue.delete(aUsedSelector)
+      if (nextQueue) {
+        nextQueue.add(aUsedSelector)
+      }
+    }
+
+    didPumpAnything = true
+    break
+  }
+
+  if (didPumpAnything) {
+    queueIsSleeping = true
+    setTimeout(() => {
+      queueIsSleeping = false
+      pumpCosmeticFilterQueues()
+    }, pumpIntervalMs)
  }
-  observer.observe(targetNode, observerConfig)
 }

 chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
  const action = typeof msg === 'string' ? msg : msg.type
  switch (action) {
    case 'cosmeticFilterGenericExceptions': {
-      let allNodes = Array.from(document.querySelectorAll('[id],[class]'))
-      handleNewNodes(allNodes)
-      applyCosmeticFilterMutationObserver()
-
+      randomizedClassName = msg.randomizedClassName
      sendResponse(null)
      break
    }
+
+    case 'cosmeticFilterConsiderNewRules': {
+      const { hideRules } = msg
+      for (const aHideRule of hideRules) {
+        if (allSelectorsSet.has(aHideRule)) {
+          continue
+        }
+        allSelectorsSet.add(aHideRule)
+        firstRunQueue.add(aHideRule)
+      }
+      pumpCosmeticFilterQueues()
+    }
  }
 })
@@ -28,10 +28,20 @@
        "https://*/*"
      ],
      "js": [
-        "out/content.bundle.js",
        "out/content_cosmetic.bundle.js"
      ],
      "run_at": "document_start",
+      "all_frames": false
+    },
+    {
+      "matches": [
+        "http://*/*",
+        "https://*/*"
+      ],
+      "js": [
+        "out/content.bundle.js"
+      ],
+      "run_at": "document_start",
      "all_frames": true
    }, {
      "matches": [
@@ -187,11 +187,12 @@ export interface GenerateClassIdStylesheet {
 interface CosmeticFilterRuleExceptionsReturn {
  type: types.COSMETIC_FILTER_RULE_EXCEPTIONS,
  tabId: number,
-  exceptions: string[]
+  exceptions: string[],
+  randomizedClassName: string
 }

 export interface CosmeticFilterRuleExceptions {
-  (tabId: number, exceptions: string[]): CosmeticFilterRuleExceptionsReturn
+  (tabId: number, exceptions: string[], randomizedClassName: string): CosmeticFilterRuleExceptionsReturn
 }

 interface ContentScriptsLoadedReturn {
@@ -338,12 +338,14 @@
    "bluebird": "^3.5.1",
    "clipboard-copy": "^2.0.0",
    "jszip": "^3.2.2",
+    "parse-domain": "^2.3.4",
    "prettier-bytes": "^1.0.4",
    "qr-image": "^3.2.0",
    "redux-logger": "^3.0.6",
    "redux-thunk": "^2.3.0",
    "throttleit": "^1.0.0",
    "unique-selector": "^0.4.1",
+    "uuid": "^3.3.2",
    "webext-redux": "^2.1.4",
    "webtorrent": "^0.107.16"
  }
@@ -33,7 +33,7 @@ function checkSelector(selector, property, expected) {
 <body>
    <div id="ad-banner"></div>
    <div class="ad-banner">
-        <div class="ad"></div>
+        <div class="ad"><img src="third-party.example.com"></img></div>
    </div>
    <div class="ad"></div>
    <div class="ad"></div>