diff --git a/changes/44746-collect-and-filter-more-cves b/changes/44746-collect-and-filter-more-cves new file mode 100644 index 0000000000..8f15ee7036 --- /dev/null +++ b/changes/44746-collect-and-filter-more-cves @@ -0,0 +1 @@ +- Started collecting non-critical CVEs, filtering them out of charts by default. \ No newline at end of file diff --git a/server/chart/api/chart.go b/server/chart/api/chart.go index 8340149a0b..202eda617b 100644 --- a/server/chart/api/chart.go +++ b/server/chart/api/chart.go @@ -85,12 +85,12 @@ type DatasetStore interface { // past the vulnerable version, so the join naturally stops matching. AffectedHostIDsByCVE(ctx context.Context, disabledFleetIDs []uint, cves []string) (map[string][]uint, error) - // TrackedCriticalCVEs returns CVE IDs matching the iteration-1 curated - // filter: critical (CVSS >= 9.0) CVEs on a hard-coded set of software - // titles, unioned with all critical OS vulnerabilities. Used by the CVE - // collector to scope collection to only the CVEs the chart actually - // renders. See TODO in the mysql implementation. - TrackedCriticalCVEs(ctx context.Context) ([]string, error) + // CollectibleCVEs returns every CVE ID, at all severities, on the curated + // set of tracked software unioned with all operating-system vulnerabilities. + // Used by the CVE collector to scope collection. Display-time narrowing + // (severity, category, EPSS, etc.) happens later at read time, so the + // collector deliberately records the wide set. See the mysql implementation. + CollectibleCVEs(ctx context.Context) ([]string, error) // RecordBucketData writes one or more entity bitmaps for the given bucket // using the specified sample strategy. See SampleStrategy for semantics. @@ -106,6 +106,21 @@ type DatasetStore interface { ) error } +// MetricCVE is the metric name of the vulnerability-exposure (CVE) dataset. +// The CVE entity filters apply only to this metric. +const MetricCVE = "cve" + +// CVE chart software category keys. These are the API contract for the +// `software_filters` query parameter and are mirrored by the frontend. The +// "os" category covers both operating-system vulnerabilities and the kernel +// software matchers. +const ( + CVECategoryOS = "os" + CVECategoryBrowsers = "browsers" + CVECategoryOffice = "office" + CVECategoryAdobe = "adobe" +) + // Host is a minimal host type for authorization checks within the chart bounded context. // The JSON tags matter: the OPA rego policy reads object.team_id via the JSON-encoded // input, so renaming or dropping the tag silently breaks team-scoped authorization. @@ -152,6 +167,21 @@ type RequestOpts struct { Platforms []string IncludeHostIDs []uint ExcludeHostIDs []uint + + // CVE entity filters (apply only to the MetricCVE metric). + SoftwareFilters []string + KnownExploit bool + // EPSS bounds are 0.0–1.0 (matching cve_meta.epss_probability); nil means + // no bound. The frontend converts its 0–100 % input before sending. + EPSSMin *float64 + EPSSMax *float64 + // Severity (CVSS) bounds are accepted but ignored this round — the service + // forces critical-only [9.0, 10.0]. See the severity TODO in the service. + SeverityMin *float64 + SeverityMax *float64 + // ExcludeCVEs is a subtractive filter — these CVEs are removed from the + // resolved entity set. + ExcludeCVEs []string } // Filters captures the applied filters for a chart request. @@ -161,4 +191,12 @@ type Filters struct { Platforms []string `json:"platforms,omitempty"` IncludeHostIDs []uint `json:"include_host_ids,omitempty"` ExcludeHostIDs []uint `json:"exclude_host_ids,omitempty"` + + SoftwareFilters []string `json:"software_filters,omitempty"` + KnownExploit bool `json:"has_known_exploit,omitempty"` + EPSSMin *float64 `json:"epss_min,omitempty"` + EPSSMax *float64 `json:"epss_max,omitempty"` + SeverityMin *float64 `json:"severity_min,omitempty"` + SeverityMax *float64 `json:"severity_max,omitempty"` + ExcludeCVEs []string `json:"exclude_vulnerabilities,omitempty"` } diff --git a/server/chart/api/http/types.go b/server/chart/api/http/types.go index 40d63074ff..192d673035 100644 --- a/server/chart/api/http/types.go +++ b/server/chart/api/http/types.go @@ -18,6 +18,17 @@ type GetChartDataRequest struct { Platforms string `query:"platforms,optional"` IncludeHostIDs string `query:"include_host_ids,optional"` ExcludeHostIDs string `query:"exclude_host_ids,optional"` + + // CVE entity filters (apply only to the cve metric). Comma-separated lists + // for categories/CVEs; EPSS and severity bounds are scalar pointers so an + // absent bound stays nil. EPSS values are 0.0–1.0. + SoftwareFilters string `query:"software_filters,optional"` + KnownExploit bool `query:"has_known_exploit,optional"` + EPSSMin *float64 `query:"epss_min,optional"` + EPSSMax *float64 `query:"epss_max,optional"` + SeverityMin *float64 `query:"severity_min,optional"` + SeverityMax *float64 `query:"severity_max,optional"` + ExcludeCVEs string `query:"exclude_vulnerabilities,optional"` } // GetChartDataResponse is the HTTP response for the chart data endpoint. diff --git a/server/chart/bootstrap/bootstrap.go b/server/chart/bootstrap/bootstrap.go index c94210f095..b6eb17015d 100644 --- a/server/chart/bootstrap/bootstrap.go +++ b/server/chart/bootstrap/bootstrap.go @@ -33,11 +33,12 @@ func New( return svc, routesFn } -// TrackedCriticalCVEs returns the curated set of CVE IDs that the chart -// collector currently tracks. Exposed for development tools (e.g. -// charts-backfill) that need to mirror the production CVE-selection logic -// without constructing the full bounded context. -func TrackedCriticalCVEs(ctx context.Context, db *sqlx.DB, logger *slog.Logger) ([]string, error) { +// CollectibleCVEs returns the wide set of CVE IDs (all severities, on the +// curated tracked software + OS vulnerabilities) that the chart collector +// records. Exposed for development tools (e.g. charts-backfill) that need to +// mirror the production CVE-selection logic without constructing the full +// bounded context. +func CollectibleCVEs(ctx context.Context, db *sqlx.DB, logger *slog.Logger) ([]string, error) { ds := mysql.NewDatastore(&platform_mysql.DBConnections{Primary: db, Replica: db}, logger) - return ds.TrackedCriticalCVEs(ctx) + return ds.CollectibleCVEs(ctx) } diff --git a/server/chart/datasets.go b/server/chart/datasets.go index 95847c9b71..11aca2d15e 100644 --- a/server/chart/datasets.go +++ b/server/chart/datasets.go @@ -34,15 +34,17 @@ func (u *UptimeDataset) Collect(ctx context.Context, store api.DatasetStore, now // CVEDataset implements api.Dataset for host CVE tracking. type CVEDataset struct{} -func (c *CVEDataset) Name() string { return "cve" } +func (c *CVEDataset) Name() string { return api.MetricCVE } func (c *CVEDataset) DefaultResolutionHours() int { return 3 } func (c *CVEDataset) SampleStrategy() api.SampleStrategy { return api.SampleStrategySnapshot } func (c *CVEDataset) DefaultVisualization() string { return "line" } func (c *CVEDataset) Collect(ctx context.Context, store api.DatasetStore, now time.Time, disabledFleetIDs []uint) error { - // Only track the CVEs that the chart API currently returns. - // TODO: implement bitmap compression so we can track all CVEs. - tracked, err := store.TrackedCriticalCVEs(ctx) + // Collect CVEs at all severities on the curated set of tracked software and + // OS vulnerabilities. Display-time narrowing (critical-only this round, + // plus user filters) happens at read time via ResolveCVEChartEntities. + // TODO: implement bitmap compression so we can track more CVEs. + tracked, err := store.CollectibleCVEs(ctx) if err != nil { return err } diff --git a/server/chart/internal/mysql/charts.go b/server/chart/internal/mysql/charts.go index b0e3966870..dac920bd74 100644 --- a/server/chart/internal/mysql/charts.go +++ b/server/chart/internal/mysql/charts.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/fleetdm/fleet/v4/server/chart/api" "github.com/fleetdm/fleet/v4/server/chart/internal/types" "github.com/fleetdm/fleet/v4/server/contexts/ctxdb" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" @@ -107,48 +108,52 @@ func (ds *Datastore) FindOnlineHostIDs(ctx context.Context, now time.Time, disab return ids, nil } -// The matcher list and TrackedCriticalCVEs exist as performance optimizations. +// The matcher list exists as a performance optimization that bounds which CVEs +// the chart collects. // TODO: implement bitmap compression so we can collect more CVE data. -// TODO: implement more filtering options for users. // cveSoftwareMatcher filters `software` rows by a MySQL LIKE pattern and an -// optional source allowlist. Empty Sources means any source. +// optional source allowlist. Empty Sources means any source. Category groups +// the matcher under one of the api.CVECategory* keys so the read-time filter +// can include/exclude whole categories. type cveSoftwareMatcher struct { + Category string NamePattern string Sources []string } -// trackedCVESoftwareMatchers is the hard-coded curated list of software -// whose critical CVEs contribute to the CVE chart. Patterns are deliberately -// broad (trailing `%`) so packaging variants (Chrome Beta/Canary, Firefox -// ESR/Nightly, kernel metapackages) are absorbed without maintenance. +// trackedCVESoftwareMatchers is the hard-coded curated list of software whose +// CVEs contribute to the CVE chart. Patterns are deliberately broad (trailing +// `%`) so packaging variants (Chrome Beta/Canary, Firefox ESR/Nightly, kernel +// metapackages) are absorbed without maintenance. The kernel matchers belong +// to the OS category alongside operating-system vulnerabilities. var trackedCVESoftwareMatchers = []cveSoftwareMatcher{ // Browsers. - {"Google Chrome%", nil}, - {"Firefox%", nil}, - {"Mozilla Firefox%", nil}, - {"Brave Browser%", nil}, - {"Safari%", []string{"apps"}}, - {"Opera%", nil}, + {api.CVECategoryBrowsers, "Google Chrome%", nil}, + {api.CVECategoryBrowsers, "Firefox%", nil}, + {api.CVECategoryBrowsers, "Mozilla Firefox%", nil}, + {api.CVECategoryBrowsers, "Brave Browser%", nil}, + {api.CVECategoryBrowsers, "Safari%", []string{"apps"}}, + {api.CVECategoryBrowsers, "Opera%", nil}, // Microsoft Office. - {"Microsoft Word%", nil}, - {"Microsoft Excel%", nil}, - {"Microsoft PowerPoint%", nil}, - {"Microsoft Outlook%", nil}, - {"Microsoft Office%", nil}, + {api.CVECategoryOffice, "Microsoft Word%", nil}, + {api.CVECategoryOffice, "Microsoft Excel%", nil}, + {api.CVECategoryOffice, "Microsoft PowerPoint%", nil}, + {api.CVECategoryOffice, "Microsoft Outlook%", nil}, + {api.CVECategoryOffice, "Microsoft Office%", nil}, // Adobe. - {"Adobe Flash%", nil}, - {"Shockwave Flash%", nil}, - {"Adobe Acrobat%", nil}, + {api.CVECategoryAdobe, "Adobe Flash%", nil}, + {api.CVECategoryAdobe, "Shockwave Flash%", nil}, + {api.CVECategoryAdobe, "Adobe Acrobat%", nil}, - // Linux kernel. Debian/Ubuntu metapackages are linux-image-* and - // linux-signed-image-*; RHEL/Fedora/Amazon Linux are kernel-* (confirmed - // via server/vulnerabilities/osv/analyzer.go rhelKernelPackages). - {"linux-image-%", []string{"deb_packages"}}, - {"linux-signed-image-%", []string{"deb_packages"}}, - {"kernel-%", []string{"rpm_packages"}}, + // Linux kernel (OS category). Debian/Ubuntu metapackages are linux-image-* + // and linux-signed-image-*; RHEL/Fedora/Amazon Linux are kernel-* + // (confirmed via server/vulnerabilities/osv/analyzer.go rhelKernelPackages). + {api.CVECategoryOS, "linux-image-%", []string{"deb_packages"}}, + {api.CVECategoryOS, "linux-signed-image-%", []string{"deb_packages"}}, + {api.CVECategoryOS, "kernel-%", []string{"rpm_packages"}}, } // AffectedHostIDsByCVE returns host IDs grouped by CVE, scoped to the given @@ -206,71 +211,180 @@ func (ds *Datastore) AffectedHostIDsByCVE(ctx context.Context, disabledFleetIDs return result, nil } -// TrackedCriticalCVEs returns the deduplicated set of CVE IDs that are -// (a) linked to any `software` row matching trackedCVESoftwareMatchers with -// `cve_meta.cvss_score >= 9.0`, OR (b) present in -// `operating_system_vulnerabilities` with `cve_meta.cvss_score >= 9.0`. +// CollectibleCVEs returns the deduplicated set of CVE IDs, at all severities, +// that are (a) linked to any `software` row matching trackedCVESoftwareMatchers, +// OR (b) present in `operating_system_vulnerabilities`. This is the wide set the +// CVE collector records; display-time severity/category/EPSS narrowing happens +// at read time via ResolveCVEChartEntities. // -// Returns a non-nil empty slice when no CVEs match, so callers can -// distinguish "filter resolved to empty" from "no filter requested" (nil). -// See GetSCDData for how empty vs nil is interpreted at the query layer. -// -// TODO: replace with user-configurable filtering. See the -// matcher-list comment above. -func (ds *Datastore) TrackedCriticalCVEs(ctx context.Context) ([]string, error) { - const criticalCVSS = 9.0 +// Returns a non-nil empty slice when no CVEs match. +func (ds *Datastore) CollectibleCVEs(ctx context.Context) ([]string, error) { set := make(map[string]struct{}) - // Software-side: build an OR-chained matcher clause. Each matcher adds one - // `(name LIKE ? AND source IN (?))` or `name LIKE ?` subclause. - softwareArgs := []any{criticalCVSS} - matcherClauses := make([]string, 0, len(trackedCVESoftwareMatchers)) - for _, m := range trackedCVESoftwareMatchers { - if len(m.Sources) == 0 { - matcherClauses = append(matcherClauses, "s.name LIKE ?") - softwareArgs = append(softwareArgs, m.NamePattern) - } else { - matcherClauses = append(matcherClauses, "(s.name LIKE ? AND s.source IN (?))") - softwareArgs = append(softwareArgs, m.NamePattern, m.Sources) + // Software-side: every tracked-matcher CVE, no cve_meta join or severity + // filter — we collect all severities and narrow only at read time. + if swClause, swArgs, ok := softwareMatcherClause(nil); ok { + swQuery := ` + SELECT DISTINCT sc.cve + FROM software_cve sc + JOIN software s ON s.id = sc.software_id + WHERE ` + swClause + expanded, expandedArgs, err := sqlx.In(swQuery, swArgs...) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "expand collectible-CVE software args") + } + expanded = ds.rebind(expanded) + if err := streamCVEStrings(ctx, ds.reader(ctx), expanded, expandedArgs, set); err != nil { + return nil, ctxerr.Wrap(ctx, err, "stream collectible-CVE software results") } } - softwareQuery := ` - SELECT DISTINCT sc.cve - FROM software_cve sc - JOIN software s ON s.id = sc.software_id - JOIN cve_meta cm ON cm.cve = sc.cve - WHERE cm.cvss_score >= ? - AND (` + strings.Join(matcherClauses, " OR ") + `)` - expanded, expandedArgs, err := sqlx.In(softwareQuery, softwareArgs...) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "expand tracked-CVE software args") - } - expanded = ds.rebind(expanded) - if err := streamCVEStrings(ctx, ds.reader(ctx), expanded, expandedArgs, set); err != nil { - return nil, ctxerr.Wrap(ctx, err, "stream tracked-CVE software results") + // OS-side: all OS vulnerabilities. Fleet's OS vuln coverage is already + // scoped to desktop OSes. + const osQuery = `SELECT DISTINCT osv.cve FROM operating_system_vulnerabilities osv` + if err := streamCVEStrings(ctx, ds.reader(ctx), osQuery, nil, set); err != nil { + return nil, ctxerr.Wrap(ctx, err, "stream collectible-CVE OS results") } - // OS-side: all OS vulnerabilities at or above the critical threshold. - // Fleet's OS vuln coverage is already scoped to desktop OSes. - const osQuery = ` - SELECT DISTINCT osv.cve - FROM operating_system_vulnerabilities osv - JOIN cve_meta cm ON cm.cve = osv.cve - WHERE cm.cvss_score >= ?` - if err := streamCVEStrings(ctx, ds.reader(ctx), osQuery, []any{criticalCVSS}, set); err != nil { - return nil, ctxerr.Wrap(ctx, err, "stream tracked-CVE OS results") + return setToSlice(set), nil +} + +// ResolveCVEChartEntities resolves the read-time CVE allow-set by intersecting +// the curated universe with the filter's predicates (software category, CVSS +// range, EPSS range, known-exploit) and subtracting any excluded CVEs. +// +// With the default filter (CVSS 9.0–10.0, all categories, no EPSS bound, no +// known-exploit, no exclusions) this reproduces the iteration-1 "tracked +// critical CVEs" set, so the chart's default display is unchanged. +// +// Returns a non-nil empty slice when the filter resolves to nothing, so callers +// never pass nil to GetSCDData (which would mean "all collected", leaking +// lower-severity CVEs into the chart). +func (ds *Datastore) ResolveCVEChartEntities(ctx context.Context, filter types.CVEChartFilter) ([]string, error) { + set := make(map[string]struct{}) + cats := categorySet(filter.Categories) // nil == all categories + metaClause, metaArgs := cveMetaPredicate(filter) + + // Software-side: skip entirely when no matcher falls in the selected + // categories (e.g. only the OS category is selected). + if swClause, swArgs, ok := softwareMatcherClause(cats); ok { + args := make([]any, 0, len(swArgs)+len(metaArgs)) + args = append(args, swArgs...) + args = append(args, metaArgs...) + swQuery := ` + SELECT DISTINCT sc.cve + FROM software_cve sc + JOIN software s ON s.id = sc.software_id + JOIN cve_meta cm ON cm.cve = sc.cve + WHERE ` + swClause + metaClause + expanded, expandedArgs, err := sqlx.In(swQuery, args...) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "expand resolve-CVE software args") + } + expanded = ds.rebind(expanded) + if err := streamCVEStrings(ctx, ds.reader(ctx), expanded, expandedArgs, set); err != nil { + return nil, ctxerr.Wrap(ctx, err, "stream resolve-CVE software results") + } } + // OS-side: only when the OS category is selected (or no category filter). + if cats == nil || containsCategory(cats, api.CVECategoryOS) { + osQuery := ` + SELECT DISTINCT osv.cve + FROM operating_system_vulnerabilities osv + JOIN cve_meta cm ON cm.cve = osv.cve + WHERE 1=1` + metaClause + expanded, expandedArgs, err := sqlx.In(osQuery, metaArgs...) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "expand resolve-CVE OS args") + } + expanded = ds.rebind(expanded) + if err := streamCVEStrings(ctx, ds.reader(ctx), expanded, expandedArgs, set); err != nil { + return nil, ctxerr.Wrap(ctx, err, "stream resolve-CVE OS results") + } + } + + // Subtract excluded CVEs. Excluding a CVE not in the set is a no-op, so a + // user may freely exclude CVEs that were never collected. + for _, cve := range filter.ExcludeCVEs { + delete(set, cve) + } + + return setToSlice(set), nil +} + +// softwareMatcherClause builds the OR-chained software matcher subclause for the +// matchers in the selected categories, plus its args. A nil categories map +// means "all categories". Returns ok=false when no matcher falls in the +// selected categories, signaling the caller to skip the software-side query. +func softwareMatcherClause(categories map[string]struct{}) (clause string, args []any, ok bool) { + subclauses := make([]string, 0, len(trackedCVESoftwareMatchers)) + for _, m := range trackedCVESoftwareMatchers { + if categories != nil { + if _, sel := categories[m.Category]; !sel { + continue + } + } + if len(m.Sources) == 0 { + subclauses = append(subclauses, "s.name LIKE ?") + args = append(args, m.NamePattern) + } else { + subclauses = append(subclauses, "(s.name LIKE ? AND s.source IN (?))") + args = append(args, m.NamePattern, m.Sources) + } + } + if len(subclauses) == 0 { + return "", nil, false + } + return "(" + strings.Join(subclauses, " OR ") + ")", args, true +} + +// cveMetaPredicate builds the cve_meta WHERE fragment (with a leading " AND ") +// shared by the software- and OS-side resolve queries, plus its args. The CVSS +// range is always applied; EPSS bounds and the known-exploit flag are optional. +func cveMetaPredicate(filter types.CVEChartFilter) (string, []any) { + clauses := []string{"cm.cvss_score >= ?", "cm.cvss_score <= ?"} + args := []any{filter.CVSSMin, filter.CVSSMax} + if filter.EPSSMin != nil { + clauses = append(clauses, "cm.epss_probability >= ?") + args = append(args, *filter.EPSSMin) + } + if filter.EPSSMax != nil { + clauses = append(clauses, "cm.epss_probability <= ?") + args = append(args, *filter.EPSSMax) + } + if filter.KnownExploit { + clauses = append(clauses, "cm.cisa_known_exploit = 1") + } + return " AND " + strings.Join(clauses, " AND "), args +} + +func categorySet(categories []string) map[string]struct{} { + if len(categories) == 0 { + return nil + } + set := make(map[string]struct{}, len(categories)) + for _, c := range categories { + set[c] = struct{}{} + } + return set +} + +func containsCategory(set map[string]struct{}, c string) bool { + _, ok := set[c] + return ok +} + +func setToSlice(set map[string]struct{}) []string { out := make([]string, 0, len(set)) for cve := range set { out = append(out, cve) } - return out, nil + return out } // streamCVEStrings runs a single-column SELECT of CVE IDs and inserts each -// into the provided set. Helper for TrackedCriticalCVEs. Streams rather than +// into the provided set. Helper for the CVE resolver queries. Streams rather than // using SelectContext so we don't materialize the full result set. func streamCVEStrings(ctx context.Context, q sqlx.QueryerContext, query string, args []any, out map[string]struct{}) error { // sqlclosecheck can't see through the QueryerContext interface to verify diff --git a/server/chart/internal/mysql/cve_filter_test.go b/server/chart/internal/mysql/cve_filter_test.go new file mode 100644 index 0000000000..dba2f866c5 --- /dev/null +++ b/server/chart/internal/mysql/cve_filter_test.go @@ -0,0 +1,172 @@ +package mysql + +import ( + "encoding/binary" + "testing" + + "github.com/fleetdm/fleet/v4/server/chart/api" + "github.com/fleetdm/fleet/v4/server/chart/internal/testutils" + "github.com/fleetdm/fleet/v4/server/chart/internal/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// swCounter feeds unique software checksums (binary(16) UNIQUE NOT NULL). +var swCounter uint64 + +// seedSoftware inserts a `software` row and a linking `software_cve` row, so a +// CVE is attributed to software of the given name+source. Returns nothing — the +// resolver/collector queries match on name/source, not id. +func seedSoftware(t *testing.T, tdb *testutils.TestDB, name, source, cve string) { + t.Helper() + ctx := t.Context() + + swCounter++ + checksum := make([]byte, 16) + binary.BigEndian.PutUint64(checksum[8:], swCounter) + + res, err := tdb.DB.ExecContext(ctx, + `INSERT INTO software (name, version, source, checksum) VALUES (?, '1.0', ?, ?)`, + name, source, checksum) + require.NoError(t, err) + swID, err := res.LastInsertId() + require.NoError(t, err) + + _, err = tdb.DB.ExecContext(ctx, + `INSERT INTO software_cve (software_id, cve) VALUES (?, ?)`, swID, cve) + require.NoError(t, err) +} + +// seedOSVuln attributes a CVE to an operating system via +// operating_system_vulnerabilities. +func seedOSVuln(t *testing.T, tdb *testutils.TestDB, osID uint, cve string) { + t.Helper() + _, err := tdb.DB.ExecContext(t.Context(), + `INSERT INTO operating_system_vulnerabilities (operating_system_id, cve) VALUES (?, ?)`, + osID, cve) + require.NoError(t, err) +} + +// seedCVEMeta inserts cve_meta for a CVE. Pass knownExploit to set the CISA flag. +func seedCVEMeta(t *testing.T, tdb *testutils.TestDB, cve string, cvss, epss float64, knownExploit bool) { + t.Helper() + _, err := tdb.DB.ExecContext(t.Context(), + `INSERT INTO cve_meta (cve, cvss_score, epss_probability, cisa_known_exploit) VALUES (?, ?, ?, ?)`, + cve, cvss, epss, knownExploit) + require.NoError(t, err) +} + +// TestCollectibleCVEs verifies the wide collection set: all severities on +// tracked software and OS vulnerabilities are collected (even without cve_meta), +// while CVEs on untracked software are excluded. +func TestCollectibleCVEs(t *testing.T) { + tdb := testutils.SetupTestDB(t, "chart_mysql") + defer tdb.TruncateTables(t) + ds := NewDatastore(tdb.Conns(), tdb.Logger) + ctx := t.Context() + + // Tracked software, low-severity CVE — collected despite low/absent severity. + seedSoftware(t, tdb, "Google Chrome", "apps", "CVE-2026-1000") + seedCVEMeta(t, tdb, "CVE-2026-1000", 3.0, 0.1, false) + // Tracked software, no cve_meta at all — still collected (severity unknown). + seedSoftware(t, tdb, "Adobe Acrobat", "programs", "CVE-2026-1001") + // OS vulnerability — collected. + seedOSVuln(t, tdb, 1, "CVE-2026-1002") + // Untracked software — NOT collected. + seedSoftware(t, tdb, "Slack", "apps", "CVE-2026-9000") + + got, err := ds.CollectibleCVEs(ctx) + require.NoError(t, err) + assert.ElementsMatch(t, []string{"CVE-2026-1000", "CVE-2026-1001", "CVE-2026-1002"}, got) + assert.NotContains(t, got, "CVE-2026-9000", "CVE on untracked software must not be collected") +} + +// TestResolveCVEChartEntities exercises the read-time narrowing across every +// filter dimension. The fixture (all critical unless noted): +// +// CVE-A Chrome (browsers) cvss 9.5 epss 0.80 kev=true +// CVE-B Firefox (browsers) cvss 3.0 epss 0.10 kev=false (low severity) +// CVE-C MS Word (office) cvss 9.9 epss 0.20 kev=false +// CVE-D kernel-x (rpm, OS) cvss 9.1 epss 0.50 kev=false +// CVE-E OS vuln (OS) cvss 9.7 epss 0.90 kev=true +// CVE-F Slack (untracked) cvss 10.0 epss 0.99 kev=true (never tracked) +func TestResolveCVEChartEntities(t *testing.T) { + tdb := testutils.SetupTestDB(t, "chart_mysql") + defer tdb.TruncateTables(t) + ds := NewDatastore(tdb.Conns(), tdb.Logger) + ctx := t.Context() + + seedSoftware(t, tdb, "Google Chrome", "apps", "CVE-A") + seedCVEMeta(t, tdb, "CVE-A", 9.5, 0.80, true) + seedSoftware(t, tdb, "Firefox", "apps", "CVE-B") + seedCVEMeta(t, tdb, "CVE-B", 3.0, 0.10, false) + seedSoftware(t, tdb, "Microsoft Word", "programs", "CVE-C") + seedCVEMeta(t, tdb, "CVE-C", 9.9, 0.20, false) + seedSoftware(t, tdb, "kernel-default", "rpm_packages", "CVE-D") + seedCVEMeta(t, tdb, "CVE-D", 9.1, 0.50, false) + seedOSVuln(t, tdb, 1, "CVE-E") + seedCVEMeta(t, tdb, "CVE-E", 9.7, 0.90, true) + seedSoftware(t, tdb, "Slack", "apps", "CVE-F") + seedCVEMeta(t, tdb, "CVE-F", 10.0, 0.99, true) + + // critical is the default service-forced severity band. + const critMin, critMax = 9.0, 10.0 + allCats := types.CVEChartFilter{CVSSMin: critMin, CVSSMax: critMax} + + cases := []struct { + name string + filter types.CVEChartFilter + want []string + }{ + { + name: "default critical, all categories", + filter: allCats, + want: []string{"CVE-A", "CVE-C", "CVE-D", "CVE-E"}, // B low-severity, F untracked + }, + { + name: "browsers only", + filter: types.CVEChartFilter{CVSSMin: critMin, CVSSMax: critMax, Categories: []string{api.CVECategoryBrowsers}}, + want: []string{"CVE-A"}, + }, + { + name: "OS category includes kernel software and OS vulns", + filter: types.CVEChartFilter{CVSSMin: critMin, CVSSMax: critMax, Categories: []string{api.CVECategoryOS}}, + want: []string{"CVE-D", "CVE-E"}, + }, + { + name: "known-exploit only", + filter: types.CVEChartFilter{CVSSMin: critMin, CVSSMax: critMax, KnownExploit: true}, + want: []string{"CVE-A", "CVE-E"}, + }, + { + name: "EPSS band 0.85-1.0", + filter: types.CVEChartFilter{CVSSMin: critMin, CVSSMax: critMax, EPSSMin: new(0.85), EPSSMax: new(1.0)}, + want: []string{"CVE-E"}, + }, + { + name: "exclude a collected CVE", + filter: types.CVEChartFilter{CVSSMin: critMin, CVSSMax: critMax, ExcludeCVEs: []string{"CVE-A"}}, + want: []string{"CVE-C", "CVE-D", "CVE-E"}, + }, + { + name: "exclude an uncollected CVE is a no-op", + filter: types.CVEChartFilter{CVSSMin: critMin, CVSSMax: critMax, ExcludeCVEs: []string{"CVE-NOT-COLLECTED"}}, + want: []string{"CVE-A", "CVE-C", "CVE-D", "CVE-E"}, + }, + { + name: "combined: browsers AND known-exploit", + filter: types.CVEChartFilter{CVSSMin: critMin, CVSSMax: critMax, Categories: []string{api.CVECategoryBrowsers}, KnownExploit: true}, + want: []string{"CVE-A"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := ds.ResolveCVEChartEntities(ctx, tc.filter) + require.NoError(t, err) + require.NotNil(t, got, "resolver must return a non-nil slice") + assert.ElementsMatch(t, tc.want, got) + assert.NotContains(t, got, "CVE-F", "untracked software CVE must never resolve") + }) + } +} diff --git a/server/chart/internal/service/handler.go b/server/chart/internal/service/handler.go index b1de99b84d..b5f01c3057 100644 --- a/server/chart/internal/service/handler.go +++ b/server/chart/internal/service/handler.go @@ -44,6 +44,14 @@ func getChartDataEndpoint(ctx context.Context, request any, svc api.Service) (pl Platforms: str.ParseStringList(req.Platforms), IncludeHostIDs: str.ParseUintList(req.IncludeHostIDs), ExcludeHostIDs: str.ParseUintList(req.ExcludeHostIDs), + + SoftwareFilters: str.ParseStringList(req.SoftwareFilters), + KnownExploit: req.KnownExploit, + EPSSMin: req.EPSSMin, + EPSSMax: req.EPSSMax, + SeverityMin: req.SeverityMin, + SeverityMax: req.SeverityMax, + ExcludeCVEs: str.ParseStringList(req.ExcludeCVEs), } resp, err := svc.GetChartData(ctx, req.Metric, opts) diff --git a/server/chart/internal/service/service.go b/server/chart/internal/service/service.go index 699dce9766..91b8843b68 100644 --- a/server/chart/internal/service/service.go +++ b/server/chart/internal/service/service.go @@ -138,10 +138,30 @@ func (s *Service) GetChartData(ctx context.Context, metric string, opts api.Requ return nil, err } + // entityIDs semantics at the storage layer: nil = no filter (all entities); + // non-nil empty = match nothing (zero-valued buckets). For the CVE metric we + // always resolve a concrete allow-set — never nil — so that lower-severity + // CVEs (now collected for all severities) never leak into the chart. var entityIDs []string - // entityIDs semantics at the storage layer: nil = no filter; non-nil empty - // = match nothing (produces zero-valued buckets). Do NOT convert empty to - // nil here. + if metric == api.MetricCVE { + // Severity is plumbed through the API (opts.SeverityMin/Max) but forced + // to critical-only this round; the severity UI lands in a follow-up. + // TODO(#47326): honor opts.SeverityMin/Max instead of hard-coding. + cveFilter := types.CVEChartFilter{ + Categories: opts.SoftwareFilters, + CVSSMin: 9.0, + CVSSMax: 10.0, + EPSSMin: opts.EPSSMin, + EPSSMax: opts.EPSSMax, + KnownExploit: opts.KnownExploit, + ExcludeCVEs: opts.ExcludeCVEs, + } + entityIDs, err = s.store.ResolveCVEChartEntities(ctx, cveFilter) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "resolve CVE chart entities") + } + } + data, err := s.store.GetSCDData(ctx, metric, startDate, endDate, bucketSize, dataset.SampleStrategy(), filterMask, entityIDs) if err != nil { return nil, err @@ -159,6 +179,16 @@ func (s *Service) GetChartData(ctx context.Context, metric string, opts api.Requ Platforms: opts.Platforms, IncludeHostIDs: opts.IncludeHostIDs, ExcludeHostIDs: opts.ExcludeHostIDs, + + SoftwareFilters: opts.SoftwareFilters, + KnownExploit: opts.KnownExploit, + EPSSMin: opts.EPSSMin, + EPSSMax: opts.EPSSMax, + // Severity is not echoed: it's forced to critical-only this round + // (see above), so echoing the client's requested severity_min/max + // would misrepresent what was actually applied. It returns to the + // echo when severity becomes a real filter (#47326). + ExcludeCVEs: opts.ExcludeCVEs, }, Data: data, }, nil diff --git a/server/chart/internal/service/service_test.go b/server/chart/internal/service/service_test.go index 88c158f932..aeccecf218 100644 --- a/server/chart/internal/service/service_test.go +++ b/server/chart/internal/service/service_test.go @@ -61,7 +61,8 @@ type mockDatastore struct { getHostIDsForFilterFunc func(ctx context.Context, hostFilter *types.HostFilter) ([]uint, error) findOnlineHostIDsFn func(ctx context.Context, now time.Time, disabledFleetIDs []uint) ([]uint, error) affectedHostIDsByCVEFn func(ctx context.Context, disabledFleetIDs []uint, cves []string) (map[string][]uint, error) - trackedCriticalCVEsFn func(ctx context.Context) ([]string, error) + collectibleCVEsFn func(ctx context.Context) ([]string, error) + resolveCVEEntitiesFn func(ctx context.Context, filter types.CVEChartFilter) ([]string, error) recordBucketDataFn func(ctx context.Context, dataset string, bucketStart time.Time, bucketSize time.Duration, strategy api.SampleStrategy, entityBitmaps map[string]*roaring.Bitmap) error recordBucketDataInvoked bool deleteAllForDatasetFn func(ctx context.Context, dataset string, batchSize int) error @@ -83,11 +84,21 @@ func (m *mockDatastore) AffectedHostIDsByCVE(ctx context.Context, disabledFleetI return nil, nil } -func (m *mockDatastore) TrackedCriticalCVEs(ctx context.Context) ([]string, error) { - if m.trackedCriticalCVEsFn != nil { - return m.trackedCriticalCVEsFn(ctx) +func (m *mockDatastore) CollectibleCVEs(ctx context.Context) ([]string, error) { + if m.collectibleCVEsFn != nil { + return m.collectibleCVEsFn(ctx) } - return nil, nil + // Match the real contract: non-nil, empty when nothing matches. + return []string{}, nil +} + +func (m *mockDatastore) ResolveCVEChartEntities(ctx context.Context, filter types.CVEChartFilter) ([]string, error) { + if m.resolveCVEEntitiesFn != nil { + return m.resolveCVEEntitiesFn(ctx, filter) + } + // Match the real contract: non-nil, empty means "match nothing" (never nil, + // which would be interpreted as "no entity filter"). + return []string{}, nil } func (m *mockDatastore) RecordBucketData(ctx context.Context, dataset string, bucketStart time.Time, bucketSize time.Duration, strategy api.SampleStrategy, entityBitmaps map[string]*roaring.Bitmap) error { @@ -299,9 +310,9 @@ func TestGetChartDataUptimePassesNilEntityIDs(t *testing.T) { svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) svc.RegisterDataset(&chart.UptimeDataset{}) - // Stub TrackedCriticalCVEs so an accidental call would fail loudly. - ds.trackedCriticalCVEsFn = func(_ context.Context) ([]string, error) { - t.Fatal("uptime path must not call TrackedCriticalCVEs") + // The uptime path must not resolve CVE entities — fail loudly if it does. + ds.resolveCVEEntitiesFn = func(_ context.Context, _ types.CVEChartFilter) ([]string, error) { + t.Fatal("uptime path must not call ResolveCVEChartEntities") return nil, nil } gotEntityIDsIsNil := false @@ -315,6 +326,89 @@ func TestGetChartDataUptimePassesNilEntityIDs(t *testing.T) { assert.True(t, gotEntityIDsIsNil, "uptime must pass nil entityIDs — the CVE branch must not leak") } +// TestGetChartDataCVEAlwaysResolvesEntities verifies the two load-bearing +// read-path guarantees for the CVE metric: (1) entity resolution always runs +// and its result is forwarded to GetSCDData as a concrete (never-nil) set, so +// newly collected lower-severity CVEs can't leak into the chart; and (2) the +// severity bounds are forced to critical [9.0, 10.0] this round regardless of +// any client-supplied severity_min/severity_max. +func TestGetChartDataCVEAlwaysResolvesEntities(t *testing.T) { + t.Run("no filters still resolves a concrete set", func(t *testing.T) { + ds := &mockDatastore{} + svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) + svc.RegisterDataset(&chart.CVEDataset{}) + + var gotFilter types.CVEChartFilter + resolveCalled := false + ds.resolveCVEEntitiesFn = func(_ context.Context, filter types.CVEChartFilter) ([]string, error) { + resolveCalled = true + gotFilter = filter + return []string{"CVE-2026-0001"}, nil + } + var gotEntityIDs []string + ds.getSCDDataFunc = func(_ context.Context, _ string, _, _ time.Time, _ time.Duration, _ api.SampleStrategy, _ *roaring.Bitmap, entityIDs []string) ([]api.DataPoint, error) { + gotEntityIDs = entityIDs + return nil, nil + } + + _, err := svc.GetChartData(t.Context(), "cve", api.RequestOpts{Days: 7}) + require.NoError(t, err) + assert.True(t, resolveCalled, "the CVE metric must always resolve its entity set") + assert.Equal(t, []string{"CVE-2026-0001"}, gotEntityIDs, "resolved set must be forwarded to GetSCDData, never nil") + assert.InDelta(t, 9.0, gotFilter.CVSSMin, 0, "severity is forced to critical") + assert.InDelta(t, 10.0, gotFilter.CVSSMax, 0, "severity is forced to critical") + }) + + t.Run("client severity bounds are overridden to critical", func(t *testing.T) { + ds := &mockDatastore{} + svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) + svc.RegisterDataset(&chart.CVEDataset{}) + + var gotFilter types.CVEChartFilter + ds.resolveCVEEntitiesFn = func(_ context.Context, filter types.CVEChartFilter) ([]string, error) { + gotFilter = filter + return []string{}, nil + } + + _, err := svc.GetChartData(t.Context(), "cve", api.RequestOpts{ + Days: 7, + SeverityMin: new(1.0), + SeverityMax: new(5.0), + }) + require.NoError(t, err) + assert.InDelta(t, 9.0, gotFilter.CVSSMin, 0, "client severity_min must be ignored this round") + assert.InDelta(t, 10.0, gotFilter.CVSSMax, 0, "client severity_max must be ignored this round") + }) + + t.Run("entity filters are forwarded to the resolver", func(t *testing.T) { + ds := &mockDatastore{} + svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) + svc.RegisterDataset(&chart.CVEDataset{}) + + var gotFilter types.CVEChartFilter + ds.resolveCVEEntitiesFn = func(_ context.Context, filter types.CVEChartFilter) ([]string, error) { + gotFilter = filter + return []string{}, nil + } + + opts := api.RequestOpts{ + Days: 7, + SoftwareFilters: []string{api.CVECategoryBrowsers, api.CVECategoryAdobe}, + KnownExploit: true, + EPSSMin: new(0.5), + EPSSMax: new(1.0), + ExcludeCVEs: []string{"CVE-2026-9999"}, + } + _, err := svc.GetChartData(t.Context(), "cve", opts) + require.NoError(t, err) + assert.Equal(t, []string{api.CVECategoryBrowsers, api.CVECategoryAdobe}, gotFilter.Categories) + assert.True(t, gotFilter.KnownExploit) + require.NotNil(t, gotFilter.EPSSMin) + assert.InDelta(t, 0.5, *gotFilter.EPSSMin, 0) + assert.Equal(t, []string{"CVE-2026-9999"}, gotFilter.ExcludeCVEs) + }) +} + func TestGetChartDataWithHostFilters(t *testing.T) { ds := &mockDatastore{} svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) @@ -549,7 +643,7 @@ func TestCollectDatasetsCVE(t *testing.T) { wantBucketStart := time.Date(2026, 4, 8, 14, 0, 0, 0, time.UTC) wantTracked := []string{"CVE-2024-0001", "CVE-2024-0002"} - ds.trackedCriticalCVEsFn = func(_ context.Context) ([]string, error) { + ds.collectibleCVEsFn = func(_ context.Context) ([]string, error) { return wantTracked, nil } var gotCVEs []string @@ -574,10 +668,10 @@ func TestCollectDatasetsCVE(t *testing.T) { err := svc.CollectDatasets(t.Context(), now, nil) require.NoError(t, err) assert.True(t, ds.recordBucketDataInvoked) - assert.Equal(t, wantTracked, gotCVEs, "TrackedCriticalCVEs result must be forwarded as the cves filter") + assert.Equal(t, wantTracked, gotCVEs, "CollectibleCVEs result must be forwarded as the cves filter") } -// TestCollectDatasetsCVEEmptyTracked verifies that when TrackedCriticalCVEs +// TestCollectDatasetsCVEEmptyTracked verifies that when CollectibleCVEs // returns an empty set, the collector still calls RecordBucketData with empty // bitmaps so recordSnapshot's "absent entities" branch can close any open // rows from prior cron ticks. Without this, dropping a CVE from the tracked @@ -587,7 +681,7 @@ func TestCollectDatasetsCVEEmptyTracked(t *testing.T) { svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) svc.RegisterDataset(&chart.CVEDataset{}) - ds.trackedCriticalCVEsFn = func(_ context.Context) ([]string, error) { + ds.collectibleCVEsFn = func(_ context.Context) ([]string, error) { return []string{}, nil } ds.affectedHostIDsByCVEFn = func(_ context.Context, _ []uint, cves []string) (map[string][]uint, error) { @@ -654,7 +748,7 @@ func TestCollectDatasetsForwardsScope(t *testing.T) { svc := NewService(&mockAuthorizer{}, ds, globalViewer(), nil) svc.RegisterDataset(&chart.CVEDataset{}) - ds.trackedCriticalCVEsFn = func(_ context.Context) ([]string, error) { + ds.collectibleCVEsFn = func(_ context.Context) ([]string, error) { return []string{"CVE-1"}, nil } var gotDisabled []uint diff --git a/server/chart/internal/testutils/testutils.go b/server/chart/internal/testutils/testutils.go index 904f2415a2..4b8fdac9ca 100644 --- a/server/chart/internal/testutils/testutils.go +++ b/server/chart/internal/testutils/testutils.go @@ -50,7 +50,8 @@ func (tdb *TestDB) Conns() *common_mysql.DBConnections { func (tdb *TestDB) TruncateTables(t *testing.T) { t.Helper() mysql_testing_utils.TruncateTables(t, tdb.DB, tdb.Logger, nil, - "host_scd_data", "hosts", "host_seen_times", "nano_enrollments", "teams") + "host_scd_data", "hosts", "host_seen_times", "nano_enrollments", "teams", + "software", "software_cve", "cve_meta", "operating_system_vulnerabilities") } // InsertSCDRow inserts a single host_scd_data row for tests. host_bitmap is diff --git a/server/chart/internal/types/chart.go b/server/chart/internal/types/chart.go index cf06a19020..b3a761ee93 100644 --- a/server/chart/internal/types/chart.go +++ b/server/chart/internal/types/chart.go @@ -28,6 +28,24 @@ type HostFilter struct { ExcludeHostIDs []uint } +// CVEChartFilter narrows the CVE chart entity set to a resolved allow-set of +// CVE IDs. All predicates AND together (intersect); ExcludeCVEs are subtracted +// afterward. Excluding a CVE that isn't in the set is a harmless no-op. +// +// Categories empty means "all categories" (no narrowing). CVSSMin/CVSSMax are +// always set by the service (forced to 9.0/10.0 this round — see the severity +// TODO in the service). EPSSMin/EPSSMax are nil when no bound was requested; +// values are 0.0–1.0 to match cve_meta.epss_probability. +type CVEChartFilter struct { + Categories []string + CVSSMin float64 + CVSSMax float64 + EPSSMin *float64 + EPSSMax *float64 + KnownExploit bool + ExcludeCVEs []string +} + // Datastore is the internal datastore interface for the chart bounded context. type Datastore interface { // FindOnlineHostIDs returns host IDs that are "online right now" per the @@ -46,14 +64,21 @@ type Datastore interface { // stops matching. AffectedHostIDsByCVE(ctx context.Context, disabledFleetIDs []uint, cves []string) (map[string][]uint, error) - // TrackedCriticalCVEs returns CVE IDs matching the iteration-1 curated - // filter: critical (CVSS >= 9.0) CVEs on a hard-coded set of software - // titles, unioned with all critical OS vulnerabilities. Returns a non-nil - // empty slice when nothing matches — callers pass this to GetSCDData's - // entityIDs parameter where nil vs empty have distinct semantics. - // - // TODO(iteration-2): replace with user-configurable filtering. - TrackedCriticalCVEs(ctx context.Context) ([]string, error) + // CollectibleCVEs returns every CVE ID, at all severities, on the curated + // set of tracked software (trackedCVESoftwareMatchers) unioned with all + // operating-system vulnerabilities. This is the wide set the CVE collector + // records into host_scd_data; display-time narrowing happens at read time + // via ResolveCVEChartEntities. Returns a non-nil empty slice when nothing + // matches. + CollectibleCVEs(ctx context.Context) ([]string, error) + + // ResolveCVEChartEntities resolves the read-time CVE allow-set for the chart + // by intersecting the curated universe with the filter's predicates + // (category, CVSS range, EPSS range, known-exploit) and subtracting any + // excluded CVEs. Returns a non-nil empty slice when the filter resolves to + // nothing — callers pass this to GetSCDData's entityIDs parameter, never + // nil, so lower-severity CVEs never leak into the chart. + ResolveCVEChartEntities(ctx context.Context, filter CVEChartFilter) ([]string, error) // RecordBucketData writes one or more entity bitmaps for the given bucket using // the specified sample strategy. See api.SampleStrategy for the semantics of diff --git a/tools/charts-backfill/main.go b/tools/charts-backfill/main.go index 91856e17cd..39e9b1c311 100644 --- a/tools/charts-backfill/main.go +++ b/tools/charts-backfill/main.go @@ -142,15 +142,15 @@ func main() { log.Fatalf("--use-tracked-cves only applies to --dataset cve (got %q)", *dataset) } ctx := context.Background() - cves, err := bootstrap.TrackedCriticalCVEs(ctx, db, slog.New(slog.DiscardHandler)) + cves, err := bootstrap.CollectibleCVEs(ctx, db, slog.New(slog.DiscardHandler)) if err != nil { - log.Fatalf("failed to query tracked CVEs: %v", err) + log.Fatalf("failed to query collectible CVEs: %v", err) } if len(cves) == 0 { - log.Fatal("tracked-CVE query returned no CVEs (vulnerability data may not be populated yet)") + log.Fatal("collectible-CVE query returned no CVEs (vulnerability data may not be populated yet)") } entityIDs = cves - log.Printf("discovered %d tracked CVEs from the live database", len(entityIDs)) + log.Printf("discovered %d collectible CVEs from the live database", len(entityIDs)) case *entityIDsStr != "": entityIDs = str.ParseStringList(*entityIDsStr) default: