diff --git a/changes/11925-flock-vuln-false-positive b/changes/11925-flock-vuln-false-positive new file mode 100644 index 0000000000..371afebc92 --- /dev/null +++ b/changes/11925-flock-vuln-false-positive @@ -0,0 +1 @@ +* fixed vulnerability mismatch between the flock browser and the discoteq/flock binary \ No newline at end of file diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index c8beee8293..4835eb3e19 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -32,8 +32,8 @@ import ( "github.com/fleetdm/fleet/v4/server/webhooks" "github.com/fleetdm/fleet/v4/server/worker" "github.com/getsentry/sentry-go" - kitlog "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" + kitlog "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/hashicorp/go-multierror" "github.com/micromdm/nanodep/godep" ) @@ -354,7 +354,7 @@ func checkNVDVulnerabilities( CPETranslationsURL: config.CPETranslationsURL, CVEFeedPrefixURL: config.CVEFeedPrefixURL, } - err := nvd.Sync(opts) + err := nvd.Sync(opts, logger) if err != nil { errHandler(ctx, logger, "syncing vulnerability database", err) // don't return, continue on ... diff --git a/docs/Using Fleet/Vulnerability-Processing.md b/docs/Using Fleet/Vulnerability-Processing.md index bca5886eff..7c7d34c68e 100644 --- a/docs/Using Fleet/Vulnerability-Processing.md +++ b/docs/Using Fleet/Vulnerability-Processing.md @@ -1,4 +1,5 @@ # Vulnerability processing + - [What to expect](#what-to-expect) - [Coverage](#coverage) - [Configuration](#configuration) @@ -42,8 +43,9 @@ detected vulnerabilities cannot be used in the same way you would use an osquery be able write a query to retrieve all CVEs). ### MacOffice release notes + We periodically parse the Mac Office release notes posted at -https://learn.microsoft.com/en-us/officeupdates/release-notes-office-for-mac and for each release + and for each release note we extract the version information and what vulnerabilities were patched with the release, we then examine all installed macOS apps and if an Office app is found we compare its version with the release notes metadata @@ -67,15 +69,18 @@ corresponding OVAL file - any match is reported using the same channels as with For Windows/Mac OS Fleet attempts to detect vulnerabilities for installed software that falls into the following categories (types): ### Apps + - macOS - Windows ### Browser plugins + - macOS, Windows - Chrome extensions - Firefox extensions ### Packages + - macOS - Python - Packages installed using Homebrew @@ -86,14 +91,17 @@ For Windows/Mac OS Fleet attempts to detect vulnerabilities for installed softwa - Packages installed using Chocolatey For Linux, we adhere to whatever is defined in the OVAL definitions, except for: + - Kernel vulnerabilities. - Vulnerabilities involving configuration files. As of right now, the following distributions are supported: + - Ubuntu - RHEL based distros (Red Hat, CentOS, Fedora, and Amazon Linux) -As of right now, only app names with all ASCII characters are supported. Apps with names featuring non-ASCII characters, such as Cyrillic, will not generate matches. +As of right now, only app names with all ASCII characters are supported. Apps with names featuring +non-ASCII characters, such as Cyrillic, will not generate matches. The ingestion of software varies per platform. For each platform, we run an [osquery query](#ingesting-software-lists-from-hosts) to ingest software. @@ -104,7 +112,7 @@ vulnerability processing and software inventory are not explicitly disabled. If you explicitly disabled vulnerability processing, and now would like to enable this feature, first enable the software inventory feature by setting the following app config: -``` +```yaml --- apiVersion: v1 kind: config @@ -116,7 +124,7 @@ spec: Then, enable vulnerability processing by specifying a path where Fleet will download the different data feeds. This can be done by setting the following app config: -``` +```yaml --- apiVersion: v1 kind: config @@ -127,7 +135,7 @@ spec: Or through environment variables: -``` +```text FLEET_VULNERABILITIES_DATABASES_PATH=/some/path ``` @@ -137,10 +145,10 @@ found in the [configuration documentation](https://fleetdm.com/docs/deploying/co You'll need to restart the Fleet instances after changing these settings. -### Advanced Configuration +### Advanced configuration Fleet runs vulnerability downloading and processing via internal scheduled cron job. This internal mechanism is very useful -for frictionless deployments and is well suited for most use cases. However, in larger deployments, +for frictionless deployments and is well suited for most use cases. However, in larger deployments, where there can be dozens of Fleet server replicas sitting behind a load balancer, it is desirable to manage vulnerability processing externally. The reasons for this are as follows: @@ -159,13 +167,17 @@ to external systems such as: - [Plain old cron](https://en.wikipedia.org/wiki/Cron) To opt into this functionality, be sure to configure your Fleet server deployment with -``` + +```bash FLEET_VULNERABILITIES_DISABLE_SCHEDULE=true ``` + which will **disable** the internal scheduling mechanism for vulnerability processing. -And then externally run with the same environment variables/configuration files passed to the server command. -``` +And then externally run with the same environment variables/configuration files passed to the server +command. + +```text fleet vuln_processing ``` @@ -178,12 +190,12 @@ instances, only one will be doing the work. In order to conduct vulnerability processing, Fleet downloads the following files: -1. A preprocessed CPE database generated by FleetDM to speed up the translation process: https://github.com/fleetdm/nvd/releases +1. A preprocessed CPE database generated by FleetDM to speed up the translation process: 2. The historical data for all CVEs and how to match to a CPE: from - https://nvd.nist.gov/vuln/data-feeds + The database generated in step 1 is processed from the original official CPE dictionary -https://nvd.nist.gov/products/cpe. This CPE dictionary is typically updated once a day. +. This CPE dictionary is typically updated once a day. The matching occurs server-side to make the processing as fast as possible, but the whole process is both CPU and memory intensive. @@ -208,6 +220,7 @@ To reduce memory footprint during the evaluation phase and because of performanc parsed, and the result is stored in a file following the following naming convention: `fleet_oval_platform_date.json`. The performance will be a function of three variables: + - The size of the OVAL file - The amount of hosts to scan - The amount of installed software @@ -225,20 +238,20 @@ The whole pipeline exists to compensate for these differences, and it can be div 1. Collection: - ```mermaid - graph TD; - host1[Host1 send software list]-->normalize[Normalization of names, versions, etc] - host2[Host2 send software list]-->normalize - host3[Host3 send software list]-->normalize - normalize-->store[Storage for later processing] - ``` + ```mermaid + graph TD; + host1[Host1 send software list]-->normalize[Normalization of names, versions, etc] + host2[Host2 send software list]-->normalize + host3[Host3 send software list]-->normalize + normalize-->store[Storage for later processing] + ``` 2. Processing - Processing happens in a loop and varies depending on the platform - first Windows/Mac OS hosts - will be processed, then we look at Linux hosts. The default interval is 1hr. + Processing happens in a loop and varies depending on the platform - first Windows/Mac OS hosts + will be processed, then we look at Linux hosts. The default interval is 1hr. - ### General process +### General process ```mermaid graph TD; @@ -257,6 +270,7 @@ The whole pipeline exists to compensate for these differences, and it can be div cpeTranslate-->cveDownload(CVE datastreams downloaded) cveDownload-->cveMap[CVE detection] ``` + ### Linux ```mermaid @@ -301,7 +315,7 @@ Let's look into some examples of this stage. tmux is a Unix terminal utility to multiplex ttys. It appears listed like this in macOS: -``` +```text osquery> SELECT * FROM homebrew_packages WHERE name='tmux'; +------+----------------------------+---------+ | name | path | version | @@ -314,7 +328,7 @@ If we look at the [official releases](https://github.com/tmux/tmux/releases/tag/ Now let's look at Chrome on macOS: -``` +```text osquery> select name, bundle_version from apps where name like '%Chrome%'; +-------------------+----------------+ | name | bundle_version | @@ -336,10 +350,11 @@ In order to improve the accuracy of matching software to CPEs, CPE translations The following CPE translation rule is used to reduce false positives when ruby is installed via homebrew. This is needed because ruby is commonly included in the title in the CPE database. -This rule matches the software name `ruby` matching a regular expression pattern and installed using `homebrew`. -When searching for CPEs, the specifed `product` and `vendor` will be added to the filter critera. +This rule matches the software name `ruby` matching a regular expression pattern and installed on +the host using `homebrew`. +When searching for [CPEs](https://en.wikipedia.org/wiki/Common_Platform_Enumeration), the specifed `product` and `vendor` will be added to the filter criteria. -``` +```json [ { "software": { @@ -353,6 +368,23 @@ When searching for CPEs, the specifed `product` and `vendor` will be added to th } ] ``` + +This translation rule maps installed homebrew packages with a name containing `ruby` with a matching. +CPE using `product == "ruby" AND vendor == "ruby-lang"` + +Take this CPE for example: + +```text +cpe:2.3:a:ruby-lang:ruby:1.8.2:*:*:*:*:*:*:* +``` + +If we do not include this translation rule, CVEs will not map correctly because Homebrew uses +version numbers in this recipe name, and does not provide a `product`: + +```text +cpe:2.3:a:ruby@1.8.2:*:1.8.2:*:*:*:*:*:*:* +``` + ##### Excluding software If software is mapped to the wrong CPE and it is known that there are no entries for it in the @@ -360,7 +392,7 @@ NVD dataset, you can specify an exclusion rule by using the `skip` field. If the software will be excluded from the NVD vulnerability scanning process and, **no NVD vulnerabilities** will be reported. -``` +```json { "software": { "name": ["Docs"], @@ -399,15 +431,21 @@ The CPE translation. Used to match CPEs in the CPE database. Fields are are AND' | Name | Type | Description | | --- | --- | --- | -| `product` | array[string] | The CPE product. | +| `product` | array[string] | The CPE product. | | `vendor` | array[string] | The CPE vendor. | | `target_sw` | array[string] | The CPE target software. | | `skip` | bool | If true, matched software will be skipped from the NVD vulnerability scanning process | ### Matching a CPE to a CVE -Once we have a good CPE, we can match it against the CVE database. We download the data streams locally and match each CPE to the whole list. The matching is done using the [nvdtools implementation](https://github.com/facebookincubator/nvdtools). +Once we have a good CPE, we can match it against the CVE database. We download the data streams +locally and match each CPE to the whole list. The matching is done using the [nvdtools +implementation](https://github.com/facebookincubator/nvdtools). + +### False positive cleanup + +False positive entries are removed during vulnerability processing if the span of time since the entry was updated is greater than 2x the [configured periodicity](https://fleetdm.com/docs/configuration/fleet-server-configuration#periodicity). - \ No newline at end of file + diff --git a/server/vulnerabilities/nvd/README.md b/server/vulnerabilities/nvd/README.md new file mode 100644 index 0000000000..5a2dc0b6f2 --- /dev/null +++ b/server/vulnerabilities/nvd/README.md @@ -0,0 +1,27 @@ +# Testing CPE Translations + +To improve accuracy when [mapping software to CVEs](../../../docs/Using%20Fleet/Vulnerability-Processing.md), we can add data to [cpe_translations.json](./cpe_translations.json) which +will get picked up by the NVD repo. + +To test these changes locally, you can: + +1. make the [appropriate](../../../docs/Using%20Fleet/Vulnerability-Processing.md#Improving-accuracy) changes to cpe_translations + +2. host this file on a local web server + + ```bash + ./tools/file-server 8082 ./server/vulnerabilities/nvd/cpe_translations.json + ``` + +3. (re)launch your local fleet server with the following `--config` + + ```yaml + vulnerabilities: + cpe_translations_url: "http://localhost:8082/cpe_translations.json" + ``` + +4. trigger the vulnerabilities scan + + ```bash + fleetctl trigger --name vulnerabilities + ``` diff --git a/server/vulnerabilities/nvd/cpe_translations.go b/server/vulnerabilities/nvd/cpe_translations.go index ee95743b4a..703858b4b0 100644 --- a/server/vulnerabilities/nvd/cpe_translations.go +++ b/server/vulnerabilities/nvd/cpe_translations.go @@ -108,10 +108,10 @@ func (r *regexpCache) Get(pattern string) (*regexp.Regexp, error) { // // [ // { -// "match": { +// "software": { // "bundle_identifier": ["com.1password.1password"] // }, -// "translation": { +// "filter": { // "product": ["1password"], // "vendor": ["agilebits"] // } diff --git a/server/vulnerabilities/nvd/cpe_translations.json b/server/vulnerabilities/nvd/cpe_translations.json index 59f75faded..2d63d3f247 100644 --- a/server/vulnerabilities/nvd/cpe_translations.json +++ b/server/vulnerabilities/nvd/cpe_translations.json @@ -100,5 +100,24 @@ "filter": { "skip": true } + }, + { + "software": { + "name": ["flock"], + "source": ["homebrew_packages"] + }, + "filter": { + "skip": true + } + }, + { + "software": { + "name": ["Flock.app"], + "source": ["apps"] + }, + "filter": { + "product": ["flock"], + "vendor": ["flock"] + } } ] diff --git a/server/vulnerabilities/nvd/sync.go b/server/vulnerabilities/nvd/sync.go index 444f225223..1266938b23 100644 --- a/server/vulnerabilities/nvd/sync.go +++ b/server/vulnerabilities/nvd/sync.go @@ -5,7 +5,6 @@ import ( "encoding/csv" "encoding/json" "fmt" - "github.com/fleetdm/fleet/v4/server/contexts/license" "io" "net/url" "os" @@ -14,6 +13,8 @@ import ( "strings" "time" + "github.com/fleetdm/fleet/v4/server/contexts/license" + "github.com/facebookincubator/nvdtools/cvefeed" feednvd "github.com/facebookincubator/nvdtools/cvefeed/nvd" "github.com/fleetdm/fleet/v4/pkg/download" @@ -32,11 +33,12 @@ type SyncOptions struct { } // Sync downloads all the vulnerability data sources. -func Sync(opts SyncOptions) error { +func Sync(opts SyncOptions, logger log.Logger) error { if err := DownloadCPEDBFromGithub(opts.VulnPath, opts.CPEDBURL); err != nil { return fmt.Errorf("sync CPE database: %w", err) } + level.Debug(logger).Log("msg", "downloading CPE translations", "url", opts.CPETranslationsURL) if err := DownloadCPETranslationsFromGithub(opts.VulnPath, opts.CPETranslationsURL); err != nil { return fmt.Errorf("sync CPE translations: %w", err) }