Flock vulnerability false positive (#13827)

changes/11925-flock-vuln-false-positive

@@ -0,0 +1 @@
+* fixed vulnerability mismatch between the flock browser and the discoteq/flock binary

cmd/fleet/cron.go

@@ -32,8 +32,8 @@ import (
 	"github.com/fleetdm/fleet/v4/server/webhooks"
 	"github.com/fleetdm/fleet/v4/server/worker"
 	"github.com/getsentry/sentry-go"
-	kitlog "github.com/go-kit/kit/log"
-	"github.com/go-kit/kit/log/level"
+	kitlog "github.com/go-kit/log"
+	"github.com/go-kit/log/level"
 	"github.com/hashicorp/go-multierror"
 	"github.com/micromdm/nanodep/godep"
 )
@@ -354,7 +354,7 @@ func checkNVDVulnerabilities(
 			CPETranslationsURL: config.CPETranslationsURL,
 			CVEFeedPrefixURL:   config.CVEFeedPrefixURL,
 		}
-		err := nvd.Sync(opts)
+		err := nvd.Sync(opts, logger)
 		if err != nil {
 			errHandler(ctx, logger, "syncing vulnerability database", err)
 			// don't return, continue on ...

docs/Using Fleet/Vulnerability-Processing.md

@@ -1,4 +1,5 @@
 # Vulnerability processing
+
 - [What to expect](#what-to-expect)
 - [Coverage](#coverage)
 - [Configuration](#configuration)
@@ -42,8 +43,9 @@ detected vulnerabilities cannot be used in the same way you would use an osquery
 be able write a query to retrieve all CVEs).
 
 ### MacOffice release notes
+
 We periodically parse the Mac Office release notes posted at
-https://learn.microsoft.com/en-us/officeupdates/release-notes-office-for-mac and for each release
+<https://learn.microsoft.com/en-us/officeupdates/release-notes-office-for-mac> and for each release
 note we extract the version
 information and what vulnerabilities were patched with the release, we then examine all installed
 macOS apps and if an Office app is found we compare its version with the release notes metadata
@@ -67,15 +69,18 @@ corresponding OVAL file - any match is reported using the same channels as with
 For Windows/Mac OS Fleet attempts to detect vulnerabilities for installed software that falls into the following categories (types):
 
 ### Apps
+
 - macOS
 - Windows
 
 ### Browser plugins
+
 - macOS, Windows
   - Chrome extensions
   - Firefox extensions
 
 ### Packages
+
 - macOS
   - Python
   - Packages installed using Homebrew
@@ -86,14 +91,17 @@ For Windows/Mac OS Fleet attempts to detect vulnerabilities for installed softwa
   - Packages installed using Chocolatey
 
 For Linux, we adhere to whatever is defined in the OVAL definitions, except for:
+
 - Kernel vulnerabilities.
 - Vulnerabilities involving configuration files.
 
 As of right now, the following distributions are supported:
+
 - Ubuntu
 - RHEL based distros (Red Hat, CentOS, Fedora, and Amazon Linux)
 
-As of right now, only app names with all ASCII characters are supported. Apps with names featuring non-ASCII characters, such as Cyrillic, will not generate matches. 
+As of right now, only app names with all ASCII characters are supported. Apps with names featuring
+non-ASCII characters, such as Cyrillic, will not generate matches.
 
 The ingestion of software varies per platform. For each platform, we run an [osquery query](#ingesting-software-lists-from-hosts) to ingest software.
 
@@ -104,7 +112,7 @@ vulnerability processing and software inventory are not explicitly disabled.
 
 If you explicitly disabled vulnerability processing, and now would like to enable this feature, first enable the software inventory feature by setting the following app config:
 
-```
+```yaml
 ---
 apiVersion: v1
 kind: config
@@ -116,7 +124,7 @@ spec:
 Then, enable vulnerability processing by specifying a path where Fleet will download the different
 data feeds. This can be done by setting the following app config:
 
-```
+```yaml
 ---
 apiVersion: v1
 kind: config
@@ -127,7 +135,7 @@ spec:
 
 Or through environment variables:
 
-```
+```text
 FLEET_VULNERABILITIES_DATABASES_PATH=/some/path
 ```
 
@@ -137,10 +145,10 @@ found in the [configuration documentation](https://fleetdm.com/docs/deploying/co
 
 You'll need to restart the Fleet instances after changing these settings.
 
-### Advanced Configuration
+### Advanced configuration
 
 Fleet runs vulnerability downloading and processing via internal scheduled cron job. This internal mechanism is very useful
-for frictionless deployments and is well suited for most use cases. However, in larger deployments, 
+for frictionless deployments and is well suited for most use cases. However, in larger deployments,
 where there can be dozens of Fleet server replicas sitting behind a load balancer, it is desirable to manage vulnerability processing externally.
 
 The reasons for this are as follows:
@@ -159,13 +167,17 @@ to external systems such as:
 - [Plain old cron](https://en.wikipedia.org/wiki/Cron)
 
 To opt into this functionality, be sure to configure your Fleet server deployment with
-```
+
+```bash
 FLEET_VULNERABILITIES_DISABLE_SCHEDULE=true
 ```
+
 which will **disable** the internal scheduling mechanism for vulnerability processing.
 
-And then externally run with the same environment variables/configuration files passed to the server command.
-```
+And then externally run with the same environment variables/configuration files passed to the server
+command.
+
+```text
 fleet vuln_processing
 ```
 
@@ -178,12 +190,12 @@ instances, only one will be doing the work.
 
 In order to conduct vulnerability processing, Fleet downloads the following files:
 
-1. A preprocessed CPE database generated by FleetDM to speed up the translation process: https://github.com/fleetdm/nvd/releases
+1. A preprocessed CPE database generated by FleetDM to speed up the translation process: <https://github.com/fleetdm/nvd/releases>
 2. The historical data for all CVEs and how to match to a CPE: from
-   https://nvd.nist.gov/vuln/data-feeds
+   <https://nvd.nist.gov/vuln/data-feeds>
 
 The database generated in step 1 is processed from the original official CPE dictionary
-https://nvd.nist.gov/products/cpe. This CPE dictionary is typically updated once a day.
+<https://nvd.nist.gov/products/cpe>. This CPE dictionary is typically updated once a day.
 
 The matching occurs server-side to make the processing as fast as possible, but the whole process is both CPU and memory intensive.
 
@@ -208,6 +220,7 @@ To reduce memory footprint during the evaluation phase and because of performanc
 parsed, and the result is stored in a file following the following naming convention: `fleet_oval_platform_date.json`.
 
 The performance will be a function of three variables:
+
 - The size of the OVAL file
 - The amount of hosts to scan
 - The amount of installed software
@@ -225,20 +238,20 @@ The whole pipeline exists to compensate for these differences, and it can be div
 
 1. Collection:
 
-  ```mermaid
-  graph TD;
-      host1[Host1 send software list]-->normalize[Normalization of names, versions, etc]
-      host2[Host2 send software list]-->normalize
-      host3[Host3 send software list]-->normalize
-      normalize-->store[Storage for later processing]
-  ```
+    ```mermaid
+    graph TD;
+        host1[Host1 send software list]-->normalize[Normalization of names, versions, etc]
+        host2[Host2 send software list]-->normalize
+        host3[Host3 send software list]-->normalize
+        normalize-->store[Storage for later processing]
+    ```
 
 2. Processing
 
-  Processing happens in a loop and varies depending on the platform - first Windows/Mac OS hosts
-  will be processed, then we look at Linux hosts. The default interval is 1hr.
+    Processing happens in a loop and varies depending on the platform - first Windows/Mac OS hosts
+    will be processed, then we look at Linux hosts. The default interval is 1hr.
 
-  ### General process
+### General process
 
   ```mermaid
   graph TD;
@@ -257,6 +270,7 @@ The whole pipeline exists to compensate for these differences, and it can be div
     cpeTranslate-->cveDownload(CVE datastreams downloaded)
     cveDownload-->cveMap[CVE detection]
   ```
+
   ### Linux
 
   ```mermaid
@@ -301,7 +315,7 @@ Let's look into some examples of this stage.
 
 tmux is a Unix terminal utility to multiplex ttys. It appears listed like this in macOS:
 
-```
+```text
 osquery> SELECT * FROM homebrew_packages WHERE name='tmux';
 +------+----------------------------+---------+
 | name | path                       | version |
@@ -314,7 +328,7 @@ If we look at the [official releases](https://github.com/tmux/tmux/releases/tag/
 
 Now let's look at Chrome on macOS:
 
-```
+```text
 osquery> select name, bundle_version from apps where name like '%Chrome%';
 +-------------------+----------------+
 | name              | bundle_version |
@@ -336,10 +350,11 @@ In order to improve the accuracy of matching software to CPEs, CPE translations
 
 The following CPE translation rule is used to reduce false positives when ruby is installed via homebrew.
 This is needed because ruby is commonly included in the title in the CPE database.
-This rule matches the software name `ruby` matching a regular expression pattern and installed using `homebrew`.
-When searching for CPEs, the specifed `product` and `vendor` will be added to the filter critera.
+This rule matches the software name `ruby` matching a regular expression pattern and installed on
+the host using `homebrew`.
+When searching for [CPEs](https://en.wikipedia.org/wiki/Common_Platform_Enumeration), the specifed `product` and `vendor` will be added to the filter criteria.
 
-```
+```json
 [
   {
     "software": {
@@ -353,6 +368,23 @@ When searching for CPEs, the specifed `product` and `vendor` will be added to th
   }
 ]
 ```
+
+This translation rule maps installed homebrew packages with a name containing `ruby` with a matching.
+CPE using `product == "ruby" AND vendor == "ruby-lang"`
+
+Take this CPE for example:
+
+```text
+cpe:2.3:a:ruby-lang:ruby:1.8.2:*:*:*:*:*:*:*
+```
+
+If we do not include this translation rule, CVEs will not map correctly because Homebrew uses
+version numbers in this recipe name, and does not provide a `product`:
+
+```text
+cpe:2.3:a:ruby@1.8.2:*:1.8.2:*:*:*:*:*:*:*
+```
+
 ##### Excluding software
 
 If software is mapped to the wrong CPE and it is known that there are no entries for it in the
@@ -360,7 +392,7 @@ NVD dataset, you can specify an exclusion rule by using the `skip` field. If the
 software will be excluded from the NVD vulnerability scanning process and, **no NVD vulnerabilities**
 will be reported.
 
-```
+```json
 {
     "software": {
       "name": ["Docs"],
@@ -399,15 +431,21 @@ The CPE translation. Used to match CPEs in the CPE database. Fields are are AND'
 
 | Name        | Type          | Description              |
 | ---         | ---           | ---                      |
-| `product`   | array[string] | The CPE product.         |
+| `product`   | array[string] | The CPE product.        |
 | `vendor`    | array[string] | The CPE vendor.          |
 | `target_sw` | array[string] | The CPE target software. |
 | `skip`      | bool          | If true, matched software will be skipped from the NVD vulnerability scanning process |
 
 ### Matching a CPE to a CVE
 
-Once we have a good CPE, we can match it against the CVE database. We download the data streams locally and match each CPE to the whole list. The matching is done using the [nvdtools implementation](https://github.com/facebookincubator/nvdtools).
+Once we have a good CPE, we can match it against the CVE database. We download the data streams
+locally and match each CPE to the whole list. The matching is done using the [nvdtools
+implementation](https://github.com/facebookincubator/nvdtools).
+
+### False positive cleanup
+
+False positive entries are removed during vulnerability processing if the span of time since the entry was updated is greater than 2x the [configured periodicity](https://fleetdm.com/docs/configuration/fleet-server-configuration#periodicity).
 
 <meta name="pageOrderInSection" value="1300">
 <meta name="description" value="Find out how Fleet detects vulnerabilities and what software it covers.">
-<meta name="navSection" value="Vuln management">
+<meta name="navSection" value="Vuln management">

server/vulnerabilities/nvd/README.md

@@ -0,0 +1,27 @@
+# Testing CPE Translations
+
+To improve accuracy when [mapping software to CVEs](../../../docs/Using%20Fleet/Vulnerability-Processing.md), we can add data to [cpe_translations.json](./cpe_translations.json) which
+will get picked up by the NVD repo.
+
+To test these changes locally, you can:
+
+1. make the [appropriate](../../../docs/Using%20Fleet/Vulnerability-Processing.md#Improving-accuracy) changes to cpe_translations
+
+2. host this file on a local web server
+
+    ```bash
+    ./tools/file-server 8082 ./server/vulnerabilities/nvd/cpe_translations.json
+    ```
+
+3. (re)launch your local fleet server with the following `--config`
+
+    ```yaml
+    vulnerabilities:
+    cpe_translations_url: "http://localhost:8082/cpe_translations.json"
+    ```
+
+4. trigger the vulnerabilities scan
+
+    ```bash
+    fleetctl trigger --name vulnerabilities
+    ```

server/vulnerabilities/nvd/cpe_translations.go

@@ -108,10 +108,10 @@ func (r *regexpCache) Get(pattern string) (*regexp.Regexp, error) {
 //
 //	[
 //	  {
-//	    "match": {
+//	    "software": {
 //	      "bundle_identifier": ["com.1password.1password"]
 //	    },
-//	    "translation": {
+//	    "filter": {
 //	      "product": ["1password"],
 //	      "vendor": ["agilebits"]
 //	    }

server/vulnerabilities/nvd/cpe_translations.json

@@ -100,5 +100,24 @@
     "filter": {
       "skip": true
     }
+  },
+  {
+    "software": {
+      "name": ["flock"],
+      "source": ["homebrew_packages"]
+    },
+    "filter": {
+      "skip": true
+    }
+  },
+  {
+    "software": {
+      "name": ["Flock.app"],
+      "source": ["apps"]
+    },
+    "filter": {
+      "product": ["flock"],
+      "vendor": ["flock"]
+    }
   }
 ]

server/vulnerabilities/nvd/sync.go

@@ -5,7 +5,6 @@ import (
 	"encoding/csv"
 	"encoding/json"
 	"fmt"
-	"github.com/fleetdm/fleet/v4/server/contexts/license"
 	"io"
 	"net/url"
 	"os"
@@ -14,6 +13,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/fleetdm/fleet/v4/server/contexts/license"
+
 	"github.com/facebookincubator/nvdtools/cvefeed"
 	feednvd "github.com/facebookincubator/nvdtools/cvefeed/nvd"
 	"github.com/fleetdm/fleet/v4/pkg/download"
@@ -32,11 +33,12 @@ type SyncOptions struct {
 }
 
 // Sync downloads all the vulnerability data sources.
-func Sync(opts SyncOptions) error {
+func Sync(opts SyncOptions, logger log.Logger) error {
 	if err := DownloadCPEDBFromGithub(opts.VulnPath, opts.CPEDBURL); err != nil {
 		return fmt.Errorf("sync CPE database: %w", err)
 	}
 
+	level.Debug(logger).Log("msg", "downloading CPE translations", "url", opts.CPETranslationsURL)
 	if err := DownloadCPETranslationsFromGithub(opts.VulnPath, opts.CPETranslationsURL); err != nil {
 		return fmt.Errorf("sync CPE translations: %w", err)
 	}