<!-- Add the related story/sub-task/bug number, like Resolves #123, or remove if NA --> **Related issue:** Resolves #44746 # Details * Adds the ability to filter historical CVE data by software type, EPSS, CVSS, CVE ID (exclude only) and "has known exploit" * Hard-codes the CVSS filter to 9.0+ for now, since that's the only data that's been collected thus far * Un-gates the collection code so that it will collect CVE data for _all_ severities (but still in the restricted set of software) Related PRs [update the front-end](https://github.com/fleetdm/fleet/pull/47674) to allow sending these filters, and [update GitOps](https://github.com/fleetdm/fleet/pull/47634) to allow changing the default filters. # Checklist for submitter If some of the following don't apply, delete the relevant line. - [X] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files) for more information. - [X] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements), JS inline code is prevented especially for url redirects, and untrusted data interpolated into shell scripts/commands is validated against shell metacharacters. ## Testing - [X] Added/updated automated tests - [X] QA'd all new/changed functionality manually ### Manual test plan — CVE chart filtering (backend smoke test) #### Setup - Premium dev server running with a few hosts carrying vulnerable software (so `cve_meta` / `software_cve` / `operating_system_vulnerabilities` are populated) - Chart data present — collector ran once, or seeded: `go run ./tools/charts-backfill --dataset cve --use-tracked-cves --days 7` - API token exported and helper set: ```bash BASE=https://localhost:8080/api/v1/fleet/charts peak() { curl -sk -H "Authorization: Bearer $TOKEN" "$BASE/$1" | jq '[.data[].value] | max'; } #### Checks (compare against the no-filter baseline) - [x] Baseline returns data — GET /charts/cve?days=7 returns a data series; .filters is empty/default - [x] Severity force-pinned to critical — cve?days=7 and cve?days=7&severity_min=0&severity_max=10 give identical peaks (no low-severity leak; client severity ignored) - [x] Category narrowing — software_categories=browsers ≤ baseline; software_categories=os,browsers,office,adobe == baseline - [x] OS category includes kernel — software_categories=os returns OS-vuln + Linux-kernel CVE counts - [x] Known-exploit narrowing — known_exploit=true ≤ baseline - [x] EPSS narrowing — epss_min=0.9 ≤ baseline; epss_min=0&epss_max=1 == baseline (EPSS is 0.0–1.0 on the API) - [x] Exclude is subtractive + tolerant — excluding a visible CVE lowers/keeps counts; exclude_cves=CVE-0000-00000 == baseline (no-op) - [x] Filters echo back — filtered requests return applied values under .filters - [x] Uptime untouched — GET /charts/uptime?days=7 returns its normal series - [x] Free-tier safety (optional) — on non-Premium, /charts/cve returns an empty series, no error - [x] > 0 rows from: SELECT COUNT(DISTINCT scd.entity_id) AS below_critical FROM host_scd_data scd JOIN cve_meta cm ON cm.cve = scd.entity_id WHERE scd.dataset='cve' AND cm.cvss_score < 9.0; - (confirms lower-severity CVEs are stored) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit ## Summary of changes * **New Features** * Added advanced CVE chart request filters: software categories, known-exploit flag, EPSS min/max, severity min/max, and excluded CVEs. * Expanded CVE chart coverage to use the full “collectible” CVE set, with filtering applied when serving chart data. * **Tests** * Added coverage for collecting collectible CVEs and resolving chart entities based on filter combinations and exclusions. * **Chores** * Updated CVE chart backfill to use collectible CVE discovery. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
296 lines
11 KiB
Go
296 lines
11 KiB
Go
// Package service provides the service implementation for the chart bounded context.
|
|
package service
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"github.com/RoaringBitmap/roaring"
|
|
"github.com/fleetdm/fleet/v4/server/chart"
|
|
"github.com/fleetdm/fleet/v4/server/chart/api"
|
|
"github.com/fleetdm/fleet/v4/server/chart/internal/types"
|
|
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
|
|
platform_authz "github.com/fleetdm/fleet/v4/server/platform/authz"
|
|
platform_http "github.com/fleetdm/fleet/v4/server/platform/http"
|
|
)
|
|
|
|
// Service is the chart bounded context service implementation.
|
|
type Service struct {
|
|
authz platform_authz.Authorizer
|
|
store types.Datastore
|
|
viewer api.ViewerProvider
|
|
datasets map[string]api.Dataset
|
|
hostCache *hostFilterCache
|
|
logger *slog.Logger
|
|
}
|
|
|
|
// NewService creates a new chart service.
|
|
func NewService(authz platform_authz.Authorizer, store types.Datastore, viewerProvider api.ViewerProvider, logger *slog.Logger) *Service {
|
|
return &Service{
|
|
authz: authz,
|
|
store: store,
|
|
viewer: viewerProvider,
|
|
datasets: make(map[string]api.Dataset),
|
|
hostCache: newHostFilterCache(hostFilterCacheTTL),
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
// Ensure Service implements api.Service at compile time.
|
|
var _ api.Service = (*Service)(nil)
|
|
|
|
func (s *Service) RegisterDataset(ds api.Dataset) {
|
|
s.datasets[ds.Name()] = ds
|
|
}
|
|
|
|
func (s *Service) CollectDatasets(ctx context.Context, now time.Time, scope api.CollectScopeFn) error {
|
|
for name, dataset := range s.datasets {
|
|
var disabledFleetIDs []uint
|
|
if scope != nil {
|
|
skip, disabled := scope(name)
|
|
if skip {
|
|
continue
|
|
}
|
|
disabledFleetIDs = disabled
|
|
}
|
|
if err := dataset.Collect(ctx, s.store, now, disabledFleetIDs); err != nil {
|
|
// Log and continue — don't let one dataset failure block others.
|
|
if s.logger != nil {
|
|
s.logger.ErrorContext(ctx, "collect chart dataset", "dataset", name, "err", ctxerr.Wrap(ctx, err, "collect chart dataset"))
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Service) GetChartData(ctx context.Context, metric string, opts api.RequestOpts) (*api.Response, error) {
|
|
// Resolve scope first: for authz we need the right action + subject, and
|
|
// for data we need the effective team set. Fail closed if there's no
|
|
// viewer — the authenticated middleware should have placed one in ctx.
|
|
isGlobal, viewerTeamIDs, err := s.viewer.ViewerScope(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Build the authz subject + action. Two distinct cases:
|
|
// - Explicit team_id: Host{TeamID: opts.TeamID} + ActionRead. Rego's
|
|
// read rule for hosts requires team_role(subject, object.team_id) to
|
|
// match, so a team user asking for a team they don't have a role on
|
|
// is rejected by policy (not by us). Global users pass via the
|
|
// global-role rules, which don't care about team_id.
|
|
// - No team_id: Host{} + ActionList. Rego's list rules pass global
|
|
// users unconditionally and pass team users who have a list-capable
|
|
// role on any of their teams. The service then scopes data below.
|
|
authzSubject := &api.Host{TeamID: opts.TeamID}
|
|
authzAction := platform_authz.ActionRead
|
|
if opts.TeamID == nil {
|
|
authzAction = platform_authz.ActionList
|
|
}
|
|
if err := s.authz.Authorize(ctx, authzSubject, authzAction); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
dataset, ok := s.datasets[metric]
|
|
if !ok {
|
|
return nil, &platform_http.BadRequestError{Message: fmt.Sprintf("unknown chart metric: %s", metric)}
|
|
}
|
|
|
|
// Don't allow requesting more days than the charts are designed to handle.
|
|
// This mostly prevents expensive queries for large day ranges.
|
|
if opts.Days < 1 || opts.Days > 31 {
|
|
return nil, &platform_http.BadRequestError{Message: fmt.Sprintf("invalid days value: %d (must be between 1 and 31)", opts.Days)}
|
|
}
|
|
|
|
// Resolution must be 0 or a positive divisor of 24.
|
|
if opts.Resolution < 0 || (opts.Resolution != 0 && 24%opts.Resolution != 0) {
|
|
return nil, &platform_http.BadRequestError{Message: fmt.Sprintf("invalid resolution value: %d (must be 0 or a positive divisor of 24)", opts.Resolution)}
|
|
}
|
|
|
|
hours := opts.Resolution
|
|
if hours <= 0 {
|
|
hours = dataset.DefaultResolutionHours()
|
|
}
|
|
bucketSize := time.Duration(hours) * time.Hour
|
|
|
|
startDate, endDate := computeBucketRange(time.Now(), bucketSize, opts.Days, opts.TZOffsetMinutes)
|
|
|
|
// Build the host filter. The bitmap mask always encodes "currently visible
|
|
// hosts" — team scoping, label/platform/include/exclude, and incidentally
|
|
// dropping hosts deleted since the SCD rows were written.
|
|
hostFilter := &types.HostFilter{
|
|
TeamIDs: effectiveTeamIDs(opts.TeamID, isGlobal, viewerTeamIDs),
|
|
LabelIDs: opts.LabelIDs,
|
|
Platforms: opts.Platforms,
|
|
IncludeHostIDs: opts.IncludeHostIDs,
|
|
ExcludeHostIDs: opts.ExcludeHostIDs,
|
|
}
|
|
|
|
filterMask, err := s.hostCache.Get(ctx, hostFilter, func(ctx context.Context) (*roaring.Bitmap, error) {
|
|
hostIDs, err := s.store.GetHostIDsForFilter(ctx, hostFilter)
|
|
if err != nil {
|
|
return nil, ctxerr.Wrap(ctx, err, "fetch host IDs for chart filter")
|
|
}
|
|
return chart.NewBitmap(hostIDs), nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// entityIDs semantics at the storage layer: nil = no filter (all entities);
|
|
// non-nil empty = match nothing (zero-valued buckets). For the CVE metric we
|
|
// always resolve a concrete allow-set — never nil — so that lower-severity
|
|
// CVEs (now collected for all severities) never leak into the chart.
|
|
var entityIDs []string
|
|
if metric == api.MetricCVE {
|
|
// Severity is plumbed through the API (opts.SeverityMin/Max) but forced
|
|
// to critical-only this round; the severity UI lands in a follow-up.
|
|
// TODO(#47326): honor opts.SeverityMin/Max instead of hard-coding.
|
|
cveFilter := types.CVEChartFilter{
|
|
Categories: opts.SoftwareFilters,
|
|
CVSSMin: 9.0,
|
|
CVSSMax: 10.0,
|
|
EPSSMin: opts.EPSSMin,
|
|
EPSSMax: opts.EPSSMax,
|
|
KnownExploit: opts.KnownExploit,
|
|
ExcludeCVEs: opts.ExcludeCVEs,
|
|
}
|
|
entityIDs, err = s.store.ResolveCVEChartEntities(ctx, cveFilter)
|
|
if err != nil {
|
|
return nil, ctxerr.Wrap(ctx, err, "resolve CVE chart entities")
|
|
}
|
|
}
|
|
|
|
data, err := s.store.GetSCDData(ctx, metric, startDate, endDate, bucketSize, dataset.SampleStrategy(), filterMask, entityIDs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &api.Response{
|
|
Metric: metric,
|
|
Visualization: dataset.DefaultVisualization(),
|
|
TotalHosts: int(chart.BlobPopcount(filterMask)), //nolint:gosec // host counts fit comfortably in int
|
|
Resolution: formatResolution(bucketSize),
|
|
Days: opts.Days,
|
|
Filters: api.Filters{
|
|
TeamID: opts.TeamID,
|
|
LabelIDs: opts.LabelIDs,
|
|
Platforms: opts.Platforms,
|
|
IncludeHostIDs: opts.IncludeHostIDs,
|
|
ExcludeHostIDs: opts.ExcludeHostIDs,
|
|
|
|
SoftwareFilters: opts.SoftwareFilters,
|
|
KnownExploit: opts.KnownExploit,
|
|
EPSSMin: opts.EPSSMin,
|
|
EPSSMax: opts.EPSSMax,
|
|
// Severity is not echoed: it's forced to critical-only this round
|
|
// (see above), so echoing the client's requested severity_min/max
|
|
// would misrepresent what was actually applied. It returns to the
|
|
// echo when severity becomes a real filter (#47326).
|
|
ExcludeCVEs: opts.ExcludeCVEs,
|
|
},
|
|
Data: data,
|
|
}, nil
|
|
}
|
|
|
|
// effectiveTeamIDs decides the team scope applied at SQL time.
|
|
//
|
|
// explicit team_id? → just that team (authz rule above already ensured
|
|
// the caller has access to it, or is global)
|
|
// global user, no team_id → nil, meaning "no team filter"
|
|
// team user, no team_id → the viewer's accessible teams. Empty-but-non-nil
|
|
// here means the user has no teams at all; SQL
|
|
// emits 1=0 so they see nothing.
|
|
func effectiveTeamIDs(requestedTeamID *uint, isGlobal bool, viewerTeamIDs []uint) []uint {
|
|
if requestedTeamID != nil {
|
|
return []uint{*requestedTeamID}
|
|
}
|
|
if isGlobal {
|
|
return nil
|
|
}
|
|
// Return a non-nil slice even when empty — the SQL builder treats non-nil
|
|
// as "scoped" and emits a no-match clause, which is what we want for a
|
|
// team user with zero team memberships.
|
|
if viewerTeamIDs == nil {
|
|
return []uint{}
|
|
}
|
|
return viewerTeamIDs
|
|
}
|
|
|
|
func (s *Service) CleanupData(ctx context.Context, days int) error {
|
|
return s.store.CleanupSCDData(ctx, days)
|
|
}
|
|
|
|
// scrubBatchSize is the upper bound on rows touched per statement during
|
|
// scrub jobs. Tunable; chosen to bound lock duration without producing
|
|
// excessive round trips on large tables.
|
|
const scrubBatchSize = 5000
|
|
|
|
func (s *Service) ScrubDatasetGlobal(ctx context.Context, dataset string) error {
|
|
return s.store.DeleteAllForDataset(ctx, dataset, scrubBatchSize)
|
|
}
|
|
|
|
func (s *Service) ScrubDatasetFleet(ctx context.Context, dataset string, fleetIDs []uint) error {
|
|
if len(fleetIDs) == 0 {
|
|
// Defensive: an enqueue with an empty fleet list shouldn't happen, but
|
|
// if it does treat as a no-op rather than scanning the entire table.
|
|
if s.logger != nil {
|
|
s.logger.WarnContext(ctx, "chart fleet scrub invoked with empty fleet list", "dataset", dataset)
|
|
}
|
|
return nil
|
|
}
|
|
hostIDs, err := s.store.HostIDsInFleets(ctx, fleetIDs)
|
|
if err != nil {
|
|
return ctxerr.Wrap(ctx, err, "load hosts for chart fleet scrub")
|
|
}
|
|
if len(hostIDs) == 0 {
|
|
// No hosts currently in any of those fleets — fleets may have been
|
|
// deleted or every host moved out before the scrub ran. Best-effort
|
|
// per design decision; no work to do.
|
|
if s.logger != nil {
|
|
s.logger.InfoContext(ctx, "chart fleet scrub: no hosts resolved", "dataset", dataset, "fleet_ids", fleetIDs)
|
|
}
|
|
return nil
|
|
}
|
|
mask := chart.NewBitmap(hostIDs)
|
|
return s.store.ApplyScrubMaskToDataset(ctx, dataset, mask, scrubBatchSize)
|
|
}
|
|
|
|
// computeBucketRange returns a (startDate, endDate) UTC pair such that the
|
|
// GetSCDData walker will emit (days*24h)/bucketSize data points labeled at
|
|
// bucket boundaries aligned to the client's local time. The last label is
|
|
// endDate — i.e., the current (possibly ongoing) bucket in the client's tz.
|
|
func computeBucketRange(now time.Time, bucketSize time.Duration, days, tzOffsetMinutes int) (time.Time, time.Time) {
|
|
loc := time.FixedZone("client", -tzOffsetMinutes*60)
|
|
localNow := now.In(loc)
|
|
|
|
var alignedEnd time.Time
|
|
if bucketSize < 24*time.Hour {
|
|
// Align to the current local bucket within the day.
|
|
step := max(int(bucketSize/time.Hour), 1)
|
|
alignedHour := (localNow.Hour() / step) * step
|
|
alignedEnd = time.Date(localNow.Year(), localNow.Month(), localNow.Day(), alignedHour, 0, 0, 0, loc)
|
|
} else {
|
|
// Daily (or coarser) — align to the start of today's local day.
|
|
alignedEnd = time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, loc)
|
|
}
|
|
|
|
endDate := alignedEnd.UTC()
|
|
startDate := endDate.Add(-time.Duration(days) * 24 * time.Hour)
|
|
return startDate, endDate
|
|
}
|
|
|
|
func formatResolution(bucketSize time.Duration) string {
|
|
switch {
|
|
case bucketSize == time.Hour:
|
|
return "hourly"
|
|
case bucketSize == 24*time.Hour:
|
|
return "daily"
|
|
case bucketSize < 24*time.Hour:
|
|
return fmt.Sprintf("%d-hour", int(bucketSize/time.Hour))
|
|
default:
|
|
return fmt.Sprintf("%d-day", int(bucketSize/(24*time.Hour)))
|
|
}
|
|
}
|