improve vuln cpe matching on macos (#6985)

* add cpe translations
* fix matching on target_sw

server/config/config.go

@@ -276,6 +276,7 @@ type VulnerabilitiesConfig struct {
 	DatabasesPath               string        `json:"databases_path" yaml:"databases_path"`
 	Periodicity                 time.Duration `json:"periodicity" yaml:"periodicity"`
 	CPEDatabaseURL              string        `json:"cpe_database_url" yaml:"cpe_database_url"`
+	CPETranslationsURL          string        `json:"cpe_translations_url" yaml:"cpe_translations_url"`
 	CVEFeedPrefixURL            string        `json:"cve_feed_prefix_url" yaml:"cve_feed_prefix_url"`
 	CurrentInstanceChecks       string        `json:"current_instance_checks" yaml:"current_instance_checks"`
 	DisableDataSync             bool          `json:"disable_data_sync" yaml:"disable_data_sync"`
@@ -641,7 +642,9 @@ func (man Manager) addConfigs() {
 	man.addConfigDuration("vulnerabilities.periodicity", 1*time.Hour,
 		"How much time to wait between processing software for vulnerabilities.")
 	man.addConfigString("vulnerabilities.cpe_database_url", "",
-		"URL from which to get the latest CPE database. If empty, defaults to the official Github link.")
+		"URL from which to get the latest CPE database. If empty, it will be downloaded from the latest release available at https://github.com/fleetdm/nvd/releases.")
+	man.addConfigString("vulnerabilities.cpe_translations_url", "",
+		"URL from which to get the latest CPE translations. If empty, it will be downloaded from the latest release available at https://github.com/fleetdm/nvd/releases.")
 	man.addConfigString("vulnerabilities.cve_feed_prefix_url", "",
 		"Prefix URL for the CVE data feed. If empty, default to https://nvd.nist.gov/")
 	man.addConfigString("vulnerabilities.current_instance_checks", "auto",
@@ -855,6 +858,7 @@ func (man Manager) LoadConfig() FleetConfig {
 			DatabasesPath:               man.getConfigString("vulnerabilities.databases_path"),
 			Periodicity:                 man.getConfigDuration("vulnerabilities.periodicity"),
 			CPEDatabaseURL:              man.getConfigString("vulnerabilities.cpe_database_url"),
+			CPETranslationsURL:          man.getConfigString("vulnerabilities.cpe_translations_url"),
 			CVEFeedPrefixURL:            man.getConfigString("vulnerabilities.cve_feed_prefix_url"),
 			CurrentInstanceChecks:       man.getConfigString("vulnerabilities.current_instance_checks"),
 			DisableDataSync:             man.getConfigBool("vulnerabilities.disable_data_sync"),

server/config/config_test.go

@@ -79,7 +79,8 @@ func TestConfigRoundtrip(t *testing.T) {
 	require.Nil(t, err)
 
 	// Ensure the read config is the same as the original
-	assert.Equal(t, *original, man.LoadConfig())
+	actual := man.LoadConfig()
+	assert.Equal(t, *original, actual)
 }
 
 func TestConfigOsqueryAsync(t *testing.T) {

server/fleet/app.go

@@ -488,6 +488,7 @@ type VulnerabilitiesConfig struct {
 	DatabasesPath               string        `json:"databases_path"`
 	Periodicity                 time.Duration `json:"periodicity"`
 	CPEDatabaseURL              string        `json:"cpe_database_url"`
+	CPETranslationsURL          string        `json:"cpe_translations_url"`
 	CVEFeedPrefixURL            string        `json:"cve_feed_prefix_url"`
 	CurrentInstanceChecks       string        `json:"current_instance_checks"`
 	DisableDataSync             bool          `json:"disable_data_sync"`

server/service/service_appconfig.go

@@ -121,6 +121,7 @@ func (svc *Service) VulnerabilitiesConfig(ctx context.Context) (*fleet.Vulnerabi
 		DatabasesPath:               svc.config.Vulnerabilities.DatabasesPath,
 		Periodicity:                 svc.config.Vulnerabilities.Periodicity,
 		CPEDatabaseURL:              svc.config.Vulnerabilities.CPEDatabaseURL,
+		CPETranslationsURL:          svc.config.Vulnerabilities.CPETranslationsURL,
 		CVEFeedPrefixURL:            svc.config.Vulnerabilities.CVEFeedPrefixURL,
 		CurrentInstanceChecks:       svc.config.Vulnerabilities.CurrentInstanceChecks,
 		DisableDataSync:             svc.config.Vulnerabilities.DisableDataSync,

server/vulnerabilities/cpe.go

@@ -2,6 +2,7 @@ package vulnerabilities
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -10,8 +11,8 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
-	"time"
 
+	"github.com/doug-martin/goqu/v9"
 	"github.com/fleetdm/fleet/v4/pkg/download"
 	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
 	"github.com/fleetdm/fleet/v4/server/fleet"
@@ -27,15 +28,7 @@ const (
 	repo  = "nvd"
 )
 
-type NVDRelease struct {
-	Etag      string
-	CreatedAt time.Time
-	CPEURL    string
-}
-
-var cpeSqliteRegex = regexp.MustCompile(`^cpe-.*\.sqlite\.gz$`)
-
-func GetLatestNVDRelease(client *http.Client) (*NVDRelease, error) {
+func GetLatestNVDRelease(client *http.Client) (*github.RepositoryRelease, error) {
 	ghclient := github.NewClient(client)
 	ctx := context.Background()
 	releases, _, err := ghclient.Repositories.ListReleases(ctx, owner, repo, &github.ListOptions{Page: 0, PerPage: 10})
@@ -43,84 +36,64 @@ func GetLatestNVDRelease(client *http.Client) (*NVDRelease, error) {
 		return nil, err
 	}
 
-	if len(releases) == 0 {
-		return nil, nil
-	}
-
-	cpeURL := ""
-
-	// TODO: get not draft release
-
-	for _, asset := range releases[0].Assets {
-		if asset != nil {
-			matched := cpeSqliteRegex.MatchString(asset.GetName())
-			if !matched {
-				continue
-			}
-			cpeURL = asset.GetBrowserDownloadURL()
+	for _, release := range releases {
+		// skip draft releases
+		if !release.GetDraft() {
+			return release, nil
 		}
 	}
 
-	return &NVDRelease{
-		Etag:      releases[0].GetName(),
-		CreatedAt: releases[0].GetCreatedAt().Time,
-		CPEURL:    cpeURL,
-	}, nil
+	return nil, errors.New("no nvd release found")
 }
 
-type syncOpts struct {
-	url string
-}
+const cpeDBFilename = "cpe.sqlite"
 
-type CPESyncOption func(*syncOpts)
+var cpeDBRegex = regexp.MustCompile(`^cpe-.*\.sqlite\.gz$`)
 
-func WithCPEURL(url string) CPESyncOption {
-	return func(o *syncOpts) {
-		o.url = url
-	}
-}
-
-const cpeDatabaseFilename = "cpe.sqlite"
-
-// DownloadCPEDatabase downloads the CPE database from the
-// latest release of github.com/fleetdm/nvd to the given dbPath.
-// An alternative URL can be set via the WithCPEURL option.
-//
-// It won't download the database if the database has already been downloaded and
-// has an mtime after the release date.
-func DownloadCPEDatabase(
+// DownloadCPEDB downloads the CPE database to the given vulnPath. If cpeDBURL is empty, attempts to download it
+// from the latest release of github.com/fleetdm/nvd. Skips downloading if CPE database is newer than the release.
+func DownloadCPEDB(
 	vulnPath string,
 	client *http.Client,
-	opts ...CPESyncOption,
+	cpeDBURL string,
 ) error {
-	var o syncOpts
-	for _, fn := range opts {
-		fn(&o)
-	}
+	path := filepath.Join(vulnPath, cpeDBFilename)
 
-	dbPath := filepath.Join(vulnPath, cpeDatabaseFilename)
-
-	if o.url == "" {
-		nvdRelease, err := GetLatestNVDRelease(client)
+	if cpeDBURL == "" {
+		release, err := GetLatestNVDRelease(client)
 		if err != nil {
 			return err
 		}
-		stat, err := os.Stat(dbPath)
-		if err != nil {
-			if !errors.Is(err, os.ErrNotExist) {
-				return err
+		stat, err := os.Stat(path)
+		switch {
+		case errors.Is(err, os.ErrNotExist):
+			// okay
+		case err != nil:
+			return err
+		default:
+			if stat.ModTime().After(release.CreatedAt.Time) {
+				// file is newer than release, do nothing
+				return nil
 			}
-		} else if !nvdRelease.CreatedAt.After(stat.ModTime()) {
-			return nil
 		}
-		o.url = nvdRelease.CPEURL
+
+		for _, asset := range release.Assets {
+			if cpeDBRegex.MatchString(asset.GetName()) {
+				cpeDBURL = asset.GetBrowserDownloadURL()
+				break
+			}
+		}
+		if cpeDBURL == "" {
+			return errors.New("failed to find cpe database in nvd release")
+
+		}
 	}
 
-	u, err := url.Parse(o.url)
+	u, err := url.Parse(cpeDBURL)
 	if err != nil {
 		return err
 	}
-	if err := download.DownloadAndExtract(client, u, dbPath); err != nil {
+	if err := download.DownloadAndExtract(client, u, path); err != nil {
 		return err
 	}
 
@@ -130,6 +103,8 @@ func DownloadCPEDatabase(
 type IndexedCPEItem struct {
 	ID         int     `json:"id" db:"rowid"`
 	Title      string  `json:"title" db:"title"`
+	Product    string  `json:"product" db:"product"`
+	Vendor     string  `json:"vendor" db:"vendor"`
 	Version    *string `json:"version" db:"version"`
 	TargetSW   *string `json:"target_sw" db:"target_sw"`
 	CPE23      string  `json:"cpe23" db:"cpe23"`
@@ -140,67 +115,387 @@ func cleanAppName(appName string) string {
 	return strings.TrimSuffix(appName, ".app")
 }
 
-var onlyAlphaNumeric = regexp.MustCompile("[^a-zA-Z0-9]+")
+var nonAlphaNumeric = regexp.MustCompile(`[^a-zA-Z0-9]+`)
 
-func CPEFromSoftware(db *sqlx.DB, software *fleet.Software) (string, error) {
-	targetSW := ""
-	switch software.Source {
-	case "apps":
-		targetSW = "macos"
-	case "python_packages":
-		targetSW = "python"
-	case "chrome_extensions":
-		targetSW = "chrome"
-	case "firefox_addons":
-		targetSW = "firefox"
-	case "safari_extensions":
-		targetSW = "safari"
-	case "deb_packages":
-	case "portage_packages":
-	case "rpm_packages":
-	case "npm_packages":
-		targetSW = `"node.js"`
-	case "atom_packages":
-	case "programs":
-		targetSW = `"windows*"`
-	case "ie_extensions":
-	case "chocolatey_packages":
-	}
+// sanitizeMatch sanitizes the search string for sqlite fts queries. Replaces all non alpha numeric characters with spaces.
+func sanitizeMatch(s string) string {
+	return nonAlphaNumeric.ReplaceAllString(s, " ")
+}
 
-	checkTargetSW := ""
-	args := []interface{}{onlyAlphaNumeric.ReplaceAllString(cleanAppName(software.Name), " ")}
-	if targetSW != "" {
-		checkTargetSW = " AND target_sw MATCH ?"
-		args = append(args, targetSW)
-	}
-	args = append(args, software.Version)
+var sanitizeVersionRe = regexp.MustCompile(`[^a-zA-Z0-9_-]+`)
 
-	query := fmt.Sprintf(
-		`SELECT rowid, * FROM cpe WHERE rowid in (
-				  SELECT rowid FROM cpe_search WHERE title MATCH ?%s
-				) and version=? order by deprecated asc`,
-		checkTargetSW,
-	)
-	var indexedCPEs []IndexedCPEItem
-	err := db.Select(&indexedCPEs, query, args...)
+// sanitizeVersion attempts to sanitize versions and attempt to make it dot separated.
+// Eg Zoom reports version as "5.11.1 (8356)". In the NVD CPE dictionary it should be 5.11.1.8356.
+func sanitizeVersion(version string) string {
+	parts := sanitizeVersionRe.Split(version, -1)
+	return strings.Trim(strings.Join(parts, "."), ".")
+}
+
+const cpeTranslationsFilename = "cpe_translations.json"
+
+func loadCPETranslations(path string) (CPETranslations, error) {
+	f, err := os.Open(path)
 	if err != nil {
-		return "", fmt.Errorf("getting cpes for: %s: %w", cleanAppName(software.Name), err)
+		return nil, err
+	}
+	defer f.Close()
+
+	var translations CPETranslations
+	if err := json.NewDecoder(f).Decode(&translations); err != nil {
+		return nil, fmt.Errorf("decode json: %w", err)
 	}
 
+	return translations, nil
+}
+
+// DownloadCPETranslations downloads the CPE translations to the given vulnPath. If cpeTranslationsURL is empty, attempts to download it
+// from the latest release of github.com/fleetdm/nvd. Skips downloading if CPE translations is newer than the release.
+func DownloadCPETranslations(vulnPath string, client *http.Client, cpeTranslationsURL string) error {
+	path := filepath.Join(vulnPath, cpeTranslationsFilename)
+
+	if cpeTranslationsURL == "" {
+		release, err := GetLatestNVDRelease(client)
+		if err != nil {
+			return err
+		}
+		stat, err := os.Stat(path)
+		switch {
+		case errors.Is(err, os.ErrNotExist):
+			// okay
+		case err != nil:
+			return err
+		default:
+			if stat.ModTime().After(release.CreatedAt.Time) {
+				// file is newer than release, do nothing
+				return nil
+			}
+		}
+
+		for _, asset := range release.Assets {
+			if cpeTranslationsFilename == asset.GetName() {
+				cpeTranslationsURL = asset.GetBrowserDownloadURL()
+				break
+			}
+		}
+		if cpeTranslationsURL == "" {
+			return errors.New("failed to find cpe translations in nvd release")
+
+		}
+	}
+
+	u, err := url.Parse(cpeTranslationsURL)
+	if err != nil {
+		return err
+	}
+	if err := download.Download(client, u, path); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// regexpCache caches compiled regular expressions. Not safe for concurrent use.
+type regexpCache struct {
+	re map[string]*regexp.Regexp
+}
+
+func newRegexpCache() *regexpCache {
+	return &regexpCache{re: make(map[string]*regexp.Regexp)}
+}
+
+func (r *regexpCache) Get(pattern string) (*regexp.Regexp, error) {
+	if re, ok := r.re[pattern]; ok {
+		return re, nil
+	}
+
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, err
+	}
+	r.re[pattern] = re
+	return re, nil
+}
+
+// CPETranslations include special case translations for software that fail to match entries in the NVD CPE Dictionary
+// using the standard logic. This may be due to unexpected vendor or product names.
+//
+// Example:
+//
+//     [
+//       {
+//         "match": {
+//           "bundle_identifier": ["com.1password.1password"]
+//         },
+//         "translation": {
+//           "product": ["1password"],
+//           "vendor": ["agilebits"]
+//         }
+//       }
+//     ]
+type CPETranslations []CPETranslationItem
+
+func (c CPETranslations) Translate(reCache *regexpCache, s *fleet.Software) (CPETranslation, bool, error) {
+	for _, item := range c {
+		match, err := item.Software.Matches(reCache, s)
+		if err != nil {
+			return CPETranslation{}, false, err
+		}
+		if match {
+			return item.Filter, true, nil
+		}
+	}
+
+	return CPETranslation{}, false, nil
+}
+
+type CPETranslationItem struct {
+	Software CPETranslationSoftware `json:"software"`
+	Filter   CPETranslation         `json:"filter"`
+}
+
+// CPETranslationSoftware represents software match criteria for cpe translations.
+type CPETranslationSoftware struct {
+	Name             []string `json:"name"`
+	BundleIdentifier []string `json:"bundle_identifier"`
+	Source           []string `json:"source"`
+}
+
+// Matches returns true if the software satifies all the match criteria.
+func (c CPETranslationSoftware) Matches(reCache *regexpCache, s *fleet.Software) (bool, error) {
+	matches := func(a, b string) (bool, error) {
+		// check if its a regular expression enclosed in '/'
+		if len(a) > 2 && a[0] == '/' && a[len(a)-1] == '/' {
+			pattern := a[1 : len(a)-1]
+			re, err := reCache.Get(pattern)
+			if err != nil {
+				return false, err
+			}
+			return re.MatchString(b), nil
+		}
+		return a == b, nil
+	}
+
+	if len(c.Name) > 0 {
+		found := false
+		for _, name := range c.Name {
+			match, err := matches(name, s.Name)
+			if err != nil {
+				return false, err
+			}
+			if match {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false, nil
+		}
+	}
+	if len(c.BundleIdentifier) > 0 {
+		found := false
+		for _, bundleID := range c.BundleIdentifier {
+			match, err := matches(bundleID, s.BundleIdentifier)
+			if err != nil {
+				return false, err
+			}
+			if match {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false, nil
+		}
+	}
+	if len(c.Source) > 0 {
+		found := false
+		for _, source := range c.Source {
+			match, err := matches(source, s.Source)
+			if err != nil {
+				return false, err
+			}
+			if match {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+type CPETranslation struct {
+	Product  []string `json:"product"`
+	Vendor   []string `json:"vendor"`
+	TargetSW []string `json:"target_sw"`
+}
+
+// CPEFromSoftware attempts to find a matching cpe entry for the given software in the NVD CPE dictionary. `db` contains data from the NVD CPE dictionary
+// and is optimized for lookups, see `GenerateCPEDB`. `translations` are used to aid in cpe matching. When searching for cpes, we first check if it matches
+// any translations, and then lookup in the cpe database based on the title, product, vendor, target_sw, and version.
+func CPEFromSoftware(db *sqlx.DB, software *fleet.Software, translations CPETranslations, reCache *regexpCache) (string, error) {
+	version := sanitizeVersion(software.Version)
+
+	ds := goqu.Dialect("sqlite").From(goqu.I("cpe_2").As("c")).
+		Select(
+			"c.rowid",
+			"c.title",
+			"c.product",
+			"c.vendor",
+			"c.version",
+			"c.target_sw",
+			"c.cpe23",
+			"c.deprecated",
+		).
+		Join(
+			goqu.I("cpe_search").As("cs"),
+			goqu.On(goqu.I("cs.rowid").Eq(goqu.I("c.rowid"))),
+		).
+		Where(
+			goqu.I("c.version").Eq(version),
+		)
+
+	translation, match, err := translations.Translate(reCache, software)
+	if err != nil {
+		return "", fmt.Errorf("translate software: %w", err)
+	}
+	if match {
+		if len(translation.Product) > 0 {
+			var exps []goqu.Expression
+			for _, product := range translation.Product {
+				exps = append(exps, goqu.I("c.product").Eq(product))
+			}
+			ds = ds.Where(goqu.Or(exps...))
+		}
+		if len(translation.Vendor) > 0 {
+			var exps []goqu.Expression
+			for _, vendor := range translation.Vendor {
+				exps = append(exps, goqu.I("c.vendor").Eq(vendor))
+			}
+			ds = ds.Where(goqu.Or(exps...))
+		}
+		if len(translation.TargetSW) > 0 {
+			var exps []goqu.Expression
+			for _, targetSW := range translation.TargetSW {
+				exps = append(exps, goqu.I("c.target_sw").Eq(targetSW))
+			}
+			ds = ds.Where(goqu.Or(exps...))
+		}
+	} else {
+
+		name := software.Name
+		var targetSW string
+
+		switch software.Source {
+		case "apps":
+			name = cleanAppName(software.Name)
+
+			// match on bundle identifier to reduce false positives for software with short names eg notes,
+			// printer, calculator.
+			// match the following target_sw
+			// - mac
+			// - mac_os
+			// - mac_os_x
+			// - macos
+			ds = ds.Where(
+				goqu.L("? LIKE '%' || c.vendor || '%'", software.BundleIdentifier),
+				goqu.Or(
+					goqu.I("c.target_sw").Eq(""),
+					goqu.I("c.target_sw").Like("mac%"),
+				),
+			)
+		case "python_packages":
+			targetSW = "python"
+		case "chrome_extensions":
+			targetSW = "chrome"
+		case "firefox_addons":
+			targetSW = "firefox"
+		case "safari_extensions":
+			targetSW = "safari"
+		case "npm_packages":
+			targetSW = `node.js`
+		case "programs":
+
+			// match the following target_sw
+			// - windows
+			// - windows_10
+			// - windows_7
+			// - windows_8
+			// - windows_8.1
+			// - windows_ce
+			// - windows_communication_foundation
+			// - windows_integrated_security
+			// - windows_mobile
+			// - windows_phone
+			// - windows_server
+			// - windows_server_2003
+			// - windows_server_2008
+			// - windows_vista
+			// - windows_xp
+			ds = ds.Where(
+				goqu.Or(
+					goqu.I("c.target_sw").Like("windows%"),
+				),
+			)
+		}
+		if targetSW != "" {
+			ds = ds.Where(
+				goqu.L("c.target_sw").Eq(targetSW),
+			)
+		}
+
+		// sanitize name for full text search on title
+		nameTerms := sanitizeMatch(name)
+		ds = ds.Where(
+			goqu.L("cs.title MATCH ?", nameTerms),
+		)
+	}
+
+	sql, args, err := ds.ToSQL()
+	if err != nil {
+		return "", fmt.Errorf("sql: %w", err)
+	}
+
+	var indexedCPEs []IndexedCPEItem
+	err = db.Select(&indexedCPEs, sql, args...)
+	if err != nil {
+		return "", fmt.Errorf("getting cpes for: %s: %w", software.Name, err)
+	}
+
+	// if there are any non-deprecated cpes, return the first one
 	for _, item := range indexedCPEs {
 		if !item.Deprecated {
 			return item.CPE23, nil
 		}
+	}
 
+	// try to find a non-deprecated cpe by looking up deprecated_by
+	for _, item := range indexedCPEs {
 		deprecatedItem := item
 		for {
 			var deprecation IndexedCPEItem
 
 			err = db.Get(
 				&deprecation,
-				`SELECT rowid, * FROM cpe c WHERE cpe23 in (
-						SELECT cpe23 from deprecated_by d where d.cpe_id=?
-					)`,
+				`
+SELECT
+    rowid,
+    title,
+    product,
+    vendor,
+    version,
+    target_sw,
+    cpe23,
+    deprecated
+FROM
+    cpe_2
+WHERE
+    cpe23 IN (
+        SELECT cpe23 FROM deprecated_by d WHERE d.cpe_id = ?
+    )
+`,
 				deprecatedItem.ID,
 			)
 			if err != nil {
@@ -224,7 +519,7 @@ func TranslateSoftwareToCPE(
 	vulnPath string,
 	logger kitlog.Logger,
 ) error {
-	dbPath := filepath.Join(vulnPath, cpeDatabaseFilename)
+	dbPath := filepath.Join(vulnPath, cpeDBFilename)
 
 	// Skip software from platforms for which we will be using OVAL for vulnerability detection.
 	iterator, err := ds.AllSoftwareWithoutCPEIterator(ctx, oval.SupportedHostPlatforms)
@@ -239,12 +534,20 @@ func TranslateSoftwareToCPE(
 	}
 	defer db.Close()
 
+	cpeTranslationsPath := filepath.Join(vulnPath, cpeTranslationsFilename)
+	cpeTranslations, err := loadCPETranslations(cpeTranslationsPath)
+	if err != nil {
+		level.Error(logger).Log("msg", "failed to load cpe translations", "err", err)
+	}
+
+	reCache := newRegexpCache()
+
 	for iterator.Next() {
 		software, err := iterator.Value()
 		if err != nil {
 			return ctxerr.Wrap(ctx, err, "getting value from iterator")
 		}
-		cpe, err := CPEFromSoftware(db, software)
+		cpe, err := CPEFromSoftware(db, software, cpeTranslations, reCache)
 		if err != nil {
 			level.Error(logger).Log("software->cpe", "error translating to CPE, skipping...", "err", err)
 			continue

server/vulnerabilities/cpe_test.go

@@ -36,22 +36,120 @@ func TestCPEFromSoftware(t *testing.T) {
 	db, err := sqliteDB(dbPath)
 	require.NoError(t, err)
 
+	reCache := newRegexpCache()
+
 	// checking an non existent version returns empty
-	cpe, err := CPEFromSoftware(db, &fleet.Software{Name: "Vendor Product-1.app", Version: "2.3.4", Source: "apps"})
+	cpe, err := CPEFromSoftware(db, &fleet.Software{Name: "Vendor Product-1.app", Version: "2.3.4", BundleIdentifier: "vendor", Source: "apps"}, nil, reCache)
 	require.NoError(t, err)
 	require.Equal(t, "", cpe)
 
 	// checking a version that exists works
-	cpe, err = CPEFromSoftware(db, &fleet.Software{Name: "Vendor Product-1.app", Version: "1.2.3", Source: "apps"})
+	cpe, err = CPEFromSoftware(db, &fleet.Software{Name: "Vendor Product-1.app", Version: "1.2.3", BundleIdentifier: "vendor", Source: "apps"}, nil, reCache)
 	require.NoError(t, err)
 	require.Equal(t, "cpe:2.3:a:vendor:product-1:1.2.3:*:*:*:*:macos:*:*", cpe)
 
 	// follows many deprecations
-	cpe, err = CPEFromSoftware(db, &fleet.Software{Name: "Vendor2 Product2.app", Version: "0.3", Source: "apps"})
+	cpe, err = CPEFromSoftware(db, &fleet.Software{Name: "Vendor2 Product2.app", Version: "0.3", BundleIdentifier: "vendor2", Source: "apps"}, nil, reCache)
 	require.NoError(t, err)
 	require.Equal(t, "cpe:2.3:a:vendor2:product4:999:*:*:*:*:macos:*:*", cpe)
 }
 
+func TestCPETranslations(t *testing.T) {
+	tempDir := t.TempDir()
+
+	items, err := cpedict.Decode(strings.NewReader(XmlCPETestDict))
+	require.NoError(t, err)
+
+	dbPath := filepath.Join(tempDir, "cpe.sqlite")
+
+	err = GenerateCPEDB(dbPath, items)
+	require.NoError(t, err)
+
+	db, err := sqliteDB(dbPath)
+	require.NoError(t, err)
+
+	tt := []struct {
+		Name         string
+		Translations CPETranslations
+		Software     *fleet.Software
+		Expected     string
+	}{
+		{
+			Name: "simple match",
+			Translations: CPETranslations{
+				{
+					Software: CPETranslationSoftware{
+						Name:   []string{"X"},
+						Source: []string{"apps"},
+					},
+					Filter: CPETranslation{
+						Product: []string{"product-1"},
+						Vendor:  []string{"vendor"},
+					},
+				},
+			},
+			Software: &fleet.Software{
+				Name:    "X",
+				Version: "1.2.3",
+				Source:  "apps",
+			},
+			Expected: "cpe:2.3:a:vendor:product-1:1.2.3:*:*:*:*:macos:*:*",
+		},
+		{
+			Name: "match name or",
+			Translations: CPETranslations{
+				{
+					Software: CPETranslationSoftware{
+						Name:   []string{"X", "Y"},
+						Source: []string{"apps"},
+					},
+					Filter: CPETranslation{
+						Product: []string{"product-1"},
+						Vendor:  []string{"vendor"},
+					},
+				},
+			},
+			Software: &fleet.Software{
+				Name:    "Y",
+				Version: "1.2.3",
+				Source:  "apps",
+			},
+			Expected: "cpe:2.3:a:vendor:product-1:1.2.3:*:*:*:*:macos:*:*",
+		},
+		{
+			Name: "match name regexp",
+			Translations: CPETranslations{
+				{
+					Software: CPETranslationSoftware{
+						Name:   []string{"/^[A-Z]$/"},
+						Source: []string{"apps"},
+					},
+					Filter: CPETranslation{
+						Product: []string{"product-1"},
+						Vendor:  []string{"vendor"},
+					},
+				},
+			},
+			Software: &fleet.Software{
+				Name:    "Z",
+				Version: "1.2.3",
+				Source:  "apps",
+			},
+			Expected: "cpe:2.3:a:vendor:product-1:1.2.3:*:*:*:*:macos:*:*",
+		},
+	}
+
+	reCache := newRegexpCache()
+
+	for _, tc := range tt {
+		t.Run(tc.Name, func(t *testing.T) {
+			cpe, err := CPEFromSoftware(db, tc.Software, tc.Translations, reCache)
+			require.NoError(t, err)
+			require.Equal(t, tc.Expected, cpe)
+		})
+	}
+}
+
 func TestSyncCPEDatabase(t *testing.T) {
 	nettest.Run(t)
 
@@ -60,7 +158,7 @@ func TestSyncCPEDatabase(t *testing.T) {
 	tempDir := t.TempDir()
 
 	// first time, db doesn't exist, so it downloads
-	err := DownloadCPEDatabase(tempDir, client)
+	err := DownloadCPEDB(tempDir, client, "")
 	require.NoError(t, err)
 
 	dbPath := filepath.Join(tempDir, "cpe.sqlite")
@@ -68,23 +166,29 @@ func TestSyncCPEDatabase(t *testing.T) {
 	require.NoError(t, err)
 
 	// and this works afterwards
-	software := &fleet.Software{Name: "1Password.app", Version: "7.2.3", Source: "apps"}
-	cpe, err := CPEFromSoftware(db, software)
+	reCache := newRegexpCache()
+
+	software := &fleet.Software{Name: "1Password.app",
+		Version:          "7.2.3",
+		BundleIdentifier: "com.1password.1password",
+		Source:           "apps",
+	}
+	cpe, err := CPEFromSoftware(db, software, nil, reCache)
 	require.NoError(t, err)
 	require.Equal(t, "cpe:2.3:a:1password:1password:7.2.3:beta0:*:*:*:macos:*:*", cpe)
 
-	npmCPE, err := CPEFromSoftware(db, &fleet.Software{Name: "Adaltas Mixme 0.4.0 for Node.js", Version: "0.4.0", Source: "npm_packages"})
+	npmCPE, err := CPEFromSoftware(db, &fleet.Software{Name: "Adaltas Mixme 0.4.0 for Node.js", Version: "0.4.0", Source: "npm_packages"}, nil, reCache)
 	require.NoError(t, err)
 	assert.Equal(t, "cpe:2.3:a:adaltas:mixme:0.4.0:*:*:*:*:node.js:*:*", npmCPE)
 
-	windowsCPE, err := CPEFromSoftware(db, &fleet.Software{Name: "HP Storage Data Protector 8.0 for Windows 8", Version: "8.0", Source: "programs"})
+	windowsCPE, err := CPEFromSoftware(db, &fleet.Software{Name: "HP Storage Data Protector 8.0 for Windows 8", Version: "8.0", Source: "programs"}, nil, reCache)
 	require.NoError(t, err)
 	assert.Equal(t, "cpe:2.3:a:hp:storage_data_protector:8.0:-:*:*:*:windows_7:*:*", windowsCPE)
 
 	// but now we truncate to make sure searching for cpe fails
 	err = os.Truncate(dbPath, 0)
 	require.NoError(t, err)
-	_, err = CPEFromSoftware(db, software)
+	_, err = CPEFromSoftware(db, software, nil, reCache)
 	require.Error(t, err)
 
 	// and we make the db older than the release
@@ -93,7 +197,7 @@ func TestSyncCPEDatabase(t *testing.T) {
 	require.NoError(t, err)
 
 	// then it will download
-	err = DownloadCPEDatabase(tempDir, client)
+	err = DownloadCPEDB(tempDir, client, "")
 	require.NoError(t, err)
 
 	// let's register the mtime for the db
@@ -106,7 +210,7 @@ func TestSyncCPEDatabase(t *testing.T) {
 	require.NoError(t, err)
 	defer db.Close()
 
-	cpe, err = CPEFromSoftware(db, software)
+	cpe, err = CPEFromSoftware(db, software, nil, reCache)
 	require.NoError(t, err)
 	require.Equal(t, "cpe:2.3:a:1password:1password:7.2.3:beta0:*:*:*:macos:*:*", cpe)
 
@@ -114,7 +218,7 @@ func TestSyncCPEDatabase(t *testing.T) {
 	time.Sleep(2 * time.Second)
 
 	// let's check it doesn't download because it's new enough
-	err = DownloadCPEDatabase(tempDir, client)
+	err = DownloadCPEDB(tempDir, client, "")
 	require.NoError(t, err)
 	stat, err = os.Stat(dbPath)
 	require.NoError(t, err)
@@ -157,16 +261,18 @@ func TestTranslateSoftwareToCPE(t *testing.T) {
 	iterator := &fakeSoftwareIterator{
 		softwares: []*fleet.Software{
 			{
-				ID:      1,
-				Name:    "Product",
-				Version: "1.2.3",
-				Source:  "apps",
+				ID:               1,
+				Name:             "Product",
+				Version:          "1.2.3",
+				BundleIdentifier: "vendor",
+				Source:           "apps",
 			},
 			{
-				ID:      2,
-				Name:    "Product2",
-				Version: "0.3",
-				Source:  "apps",
+				ID:               2,
+				Name:             "Product2",
+				Version:          "0.3",
+				BundleIdentifier: "vendor2",
+				Source:           "apps",
 			},
 		},
 	}
@@ -204,7 +310,7 @@ func TestSyncsCPEFromURL(t *testing.T) {
 
 	client := fleethttp.NewClient()
 	tempDir := t.TempDir()
-	err := DownloadCPEDatabase(tempDir, client, WithCPEURL(ts.URL+"/hello-world.gz"))
+	err := DownloadCPEDB(tempDir, client, ts.URL+"/hello-world.gz")
 	require.NoError(t, err)
 
 	dbPath := filepath.Join(tempDir, "cpe.sqlite")
@@ -212,3 +318,39 @@ func TestSyncsCPEFromURL(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "Hello world!", string(stored))
 }
+
+func TestLegacyCPEDB(t *testing.T) {
+
+	// Older versions of fleet used "select * ..." when querying from the cpe and cpe_search tables
+	// Ensure that this still works when generating the new cpe database.
+	type IndexedCPEItem struct {
+		ID         int     `json:"id" db:"rowid"`
+		Title      string  `json:"title" db:"title"`
+		Version    *string `json:"version" db:"version"`
+		TargetSW   *string `json:"target_sw" db:"target_sw"`
+		CPE23      string  `json:"cpe23" db:"cpe23"`
+		Deprecated bool    `json:"deprecated" db:"deprecated"`
+	}
+	tempDir := t.TempDir()
+
+	items, err := cpedict.Decode(strings.NewReader(XmlCPETestDict))
+	require.NoError(t, err)
+
+	dbPath := filepath.Join(tempDir, "cpe.sqlite")
+
+	err = GenerateCPEDB(dbPath, items)
+	require.NoError(t, err)
+
+	db, err := sqliteDB(dbPath)
+	require.NoError(t, err)
+
+	query := `SELECT rowid, * FROM cpe WHERE rowid in (
+				  SELECT rowid FROM cpe_search WHERE title MATCH ?
+				) and version = ? order by deprecated asc`
+
+	var indexedCPEs []IndexedCPEItem
+	err = db.Select(&indexedCPEs, query, "product", "1.2.3")
+	require.NoError(t, err)
+
+	require.Len(t, indexedCPEs, 1)
+}

server/vulnerabilities/cpe_translations.json

@@ -0,0 +1,33 @@
+[
+  {
+    "software": {
+      "bundle_identifier": ["us.zoom.xos"],
+      "source": ["apps"]
+    },
+    "filter": {
+      "product": ["zoom", "meetings"],
+      "vendor": ["zoom"],
+      "target_sw": ["macos", "mac_os"]
+    }
+  },
+  {
+    "software": {
+      "name": ["/^ruby(@.*)?$/"],
+      "source": ["homebrew_packages"]
+    },
+    "filter": {
+      "product": ["ruby"],
+      "vendor": ["ruby-lang"]
+    }
+  },
+  {
+    "software": {
+      "name": ["/^node(@.*)?$/"],
+      "source": ["homebrew_packages"]
+    },
+    "filter": {
+      "product": ["node.js"],
+      "vendor": ["nodejs"]
+    }
+  }
+]

server/vulnerabilities/db.go

@@ -21,25 +21,41 @@ func sqliteDB(dbPath string) (*sqlx.DB, error) {
 }
 
 func applyCPEDatabaseSchema(db *sqlx.DB) error {
+	// Use a new table cpe_2 containing new columns vendor, product. view cpe used for backwards compatibility
+	// with old fleet versions that use "select * from cpe ...". When creating the view, we need to
+	// select rowid because it is used for joins between the cpe and cpe_search tables
 	_, err := db.Exec(`
-	CREATE TABLE IF NOT EXISTS cpe (
-		cpe23 TEXT NOT NULL,
-		title TEXT NOT NULL,
-		version TEXT,
-		target_sw TEXT,
-		deprecated BOOLEAN DEFAULT FALSE
-	);
-	CREATE TABLE IF NOT EXISTS deprecated_by (
-		cpe_id INTEGER,
-		cpe23 TEXT NOT NULL,
-		FOREIGN KEY(cpe_id) REFERENCES cpe(rowid)
-	);
-	CREATE VIRTUAL TABLE IF NOT EXISTS cpe_search USING fts5(title, target_sw);
-	CREATE INDEX IF NOT EXISTS idx_version ON cpe (version);
-	CREATE INDEX IF NOT EXISTS idx_cpe23 ON cpe (cpe23);
-	CREATE INDEX IF NOT EXISTS idx_target_sw ON cpe (target_sw);
-	CREATE INDEX IF NOT EXISTS idx_deprecated_by ON deprecated_by (cpe23);
-	`)
+CREATE TABLE IF NOT EXISTS cpe_2 (
+    cpe23 TEXT NOT NULL,
+    title TEXT NOT NULL,
+    vendor TEXT,
+    product TEXT,
+    version TEXT,
+    target_sw TEXT,
+    deprecated BOOLEAN DEFAULT FALSE
+);
+CREATE VIEW IF NOT EXISTS cpe AS
+SELECT
+    rowid,
+    cpe23,
+    title,
+    version,
+    target_sw,
+    deprecated
+FROM cpe_2;
+CREATE TABLE IF NOT EXISTS deprecated_by (
+    cpe_id INTEGER,
+    cpe23 TEXT NOT NULL,
+    FOREIGN KEY(cpe_id) REFERENCES cpe(rowid)
+);
+CREATE VIRTUAL TABLE IF NOT EXISTS cpe_search USING fts5(title, target_sw);
+CREATE INDEX IF NOT EXISTS idx_cpe_2_cpe23 ON cpe_2 (cpe23);
+CREATE INDEX IF NOT EXISTS idx_cpe_2_vendor ON cpe_2 (vendor);
+CREATE INDEX IF NOT EXISTS idx_cpe_2_product ON cpe_2 (product);
+CREATE INDEX IF NOT EXISTS idx_cpe_2_version ON cpe_2 (version);
+CREATE INDEX IF NOT EXISTS idx_cpe_2_target_sw ON cpe_2 (target_sw);
+CREATE INDEX IF NOT EXISTS idx_deprecated_by ON deprecated_by (cpe23);
+`)
 	return err
 }
 
@@ -47,11 +63,14 @@ func generateCPEItem(item cpedict.CPEItem) ([]interface{}, map[string]string, er
 	var cpes []interface{}
 	deprecations := make(map[string]string)
 
-	targetSW := wfn.StripSlashes(item.CPE23.Name.TargetSW)
-	version := wfn.StripSlashes(item.CPE23.Name.Version)
-	title := item.Title["en-US"]
 	cpe23 := wfn.Attributes(item.CPE23.Name).BindToFmtString()
-	cpes = append(cpes, cpe23, title, version, targetSW, item.Deprecated)
+	title := item.Title["en-US"]
+	vendor := wfn.StripSlashes(item.CPE23.Name.Vendor)
+	product := wfn.StripSlashes(item.CPE23.Name.Product)
+	version := wfn.StripSlashes(item.CPE23.Name.Version)
+	targetSW := wfn.StripSlashes(item.CPE23.Name.TargetSW)
+
+	cpes = append(cpes, cpe23, title, vendor, product, version, targetSW, item.Deprecated)
 
 	if item.CPE23.Deprecation != nil {
 		for _, deprecatedBy := range item.CPE23.Deprecation.DeprecatedBy {
@@ -82,54 +101,54 @@ func GenerateCPEDB(path string, items *cpedict.CPEList) error {
 	}
 
 	cpesCount := 0
-	var allCPEs []interface{}
+	var cpesBatch []interface{}
 	deprecationsCount := 0
-	var allDeprecations []interface{}
+	var deprecationsBatch []interface{}
 
 	for _, item := range items.Items {
 		cpes, deprecations, err := generateCPEItem(item)
 		if err != nil {
 			return err
 		}
-		allCPEs = append(allCPEs, cpes...)
+		cpesBatch = append(cpesBatch, cpes...)
 		cpesCount++
 		if len(deprecations) > 0 {
 			deprecationsCount++
 		}
 		for key, val := range deprecations {
-			allDeprecations = append(allDeprecations, key, val)
+			deprecationsBatch = append(deprecationsBatch, key, val)
 		}
 		if cpesCount > batchSize {
-			err = bulkInsertCPEs(cpesCount, db, allCPEs)
+			err = bulkInsertCPEs(cpesCount, db, cpesBatch)
 			if err != nil {
 				return err
 			}
-			allCPEs = []interface{}{}
+			cpesBatch = []interface{}{}
 			cpesCount = 0
 		}
 		if deprecationsCount > batchSize {
-			err := bulkInsertDeprecations(deprecationsCount, db, allDeprecations)
+			err := bulkInsertDeprecations(deprecationsCount, db, deprecationsBatch)
 			if err != nil {
 				return err
 			}
-			allDeprecations = []interface{}{}
+			deprecationsBatch = []interface{}{}
 			deprecationsCount = 0
 		}
 	}
 	if cpesCount > 0 {
-		err = bulkInsertCPEs(cpesCount, db, allCPEs)
+		err = bulkInsertCPEs(cpesCount, db, cpesBatch)
 		if err != nil {
 			return err
 		}
 	}
 	if deprecationsCount > 0 {
-		err := bulkInsertDeprecations(deprecationsCount, db, allDeprecations)
+		err := bulkInsertDeprecations(deprecationsCount, db, deprecationsBatch)
 		if err != nil {
 			return err
 		}
 	}
 
-	_, err = db.Exec(`INSERT INTO cpe_search(rowid, title, target_sw) select rowid, title, target_sw from cpe`)
+	_, err = db.Exec(`INSERT INTO cpe_search (rowid, title, target_sw) select rowid, title, target_sw from cpe`)
 	if err != nil {
 		return err
 	}
@@ -137,7 +156,7 @@ func GenerateCPEDB(path string, items *cpedict.CPEList) error {
 }
 
 func bulkInsertDeprecations(deprecationsCount int, db *sqlx.DB, allDeprecations []interface{}) error {
-	values := strings.TrimSuffix(strings.Repeat("((SELECT rowid FROM CPE where cpe23=?), ?),", deprecationsCount), ",")
+	values := strings.TrimSuffix(strings.Repeat("((SELECT rowid FROM CPE where cpe23 = ?), ?),", deprecationsCount), ",")
 	_, err := db.Exec(
 		fmt.Sprintf(`INSERT INTO deprecated_by(cpe_id, cpe23) VALUES %s`, values),
 		allDeprecations...,
@@ -146,9 +165,19 @@ func bulkInsertDeprecations(deprecationsCount int, db *sqlx.DB, allDeprecations
 }
 
 func bulkInsertCPEs(cpesCount int, db *sqlx.DB, allCPEs []interface{}) error {
-	values := strings.TrimSuffix(strings.Repeat("(?, ?, ?, ?, ?),", cpesCount), ",")
+	values := strings.TrimSuffix(strings.Repeat("(?, ?, ?, ?, ?, ?, ?), ", cpesCount), ", ")
 	_, err := db.Exec(
-		fmt.Sprintf(`INSERT INTO cpe(cpe23, title, version, target_sw, deprecated) VALUES %s`, values),
+		fmt.Sprintf(`
+INSERT INTO cpe_2 (
+	cpe23,
+	title,
+	vendor,
+	product,
+	version,
+	target_sw,
+	deprecated
+)
+VALUES %s`, values),
 		allCPEs...,
 	)
 	return err

server/vulnerabilities/sync.go

@@ -24,23 +24,34 @@ import (
 	"github.com/go-kit/kit/log/level"
 )
 
+type SyncOptions struct {
+	VulnPath           string
+	CPEDBURL           string
+	CPETranslationsURL string
+	CVEFeedPrefixURL   string
+}
+
 // Sync downloads all the vulnerability data sources.
-func Sync(vulnPath string, cpeDatabaseURL string) error {
+func Sync(opts SyncOptions) error {
 	client := fleethttp.NewClient()
 
-	if err := DownloadCPEDatabase(vulnPath, client, WithCPEURL(cpeDatabaseURL)); err != nil {
+	if err := DownloadCPEDB(opts.VulnPath, client, opts.CPEDBURL); err != nil {
 		return fmt.Errorf("sync CPE database: %w", err)
 	}
 
-	if err := DownloadNVDCVEFeed(vulnPath, ""); err != nil {
+	if err := DownloadCPETranslations(opts.VulnPath, client, opts.CPETranslationsURL); err != nil {
+		return fmt.Errorf("sync CPE translations: %w", err)
+	}
+
+	if err := DownloadNVDCVEFeed(opts.VulnPath, opts.CVEFeedPrefixURL); err != nil {
 		return fmt.Errorf("sync NVD CVE feed: %w", err)
 	}
 
-	if err := DownloadEPSSFeed(vulnPath, client); err != nil {
+	if err := DownloadEPSSFeed(opts.VulnPath, client); err != nil {
 		return fmt.Errorf("sync EPSS CVE feed: %w", err)
 	}
 
-	if err := DownloadCISAKnownExploitsFeed(vulnPath, client); err != nil {
+	if err := DownloadCISAKnownExploitsFeed(opts.VulnPath, client); err != nil {
 		return fmt.Errorf("sync CISA known exploits feed: %w", err)
 	}
 

server/vulnerabilities/sync_test.go

@@ -71,3 +71,16 @@ func TestLoadCVEMeta(t *testing.T) {
 	require.Equal(t, float64(0.01843), *meta.EPSSProbability)
 	require.Equal(t, true, *meta.CISAKnownExploit)
 }
+
+func TestDownloadCPETranslations(t *testing.T) {
+	nettest.Run(t)
+
+	client := fleethttp.NewClient()
+
+	tempDir := t.TempDir()
+
+	err := DownloadCPETranslations(tempDir, client, "")
+	require.NoError(t, err)
+
+	assert.FileExists(t, filepath.Join(tempDir, cpeTranslationsFilename))
+}