From 53e112d2649a0972ea594a055f6a2c96d74d53a4 Mon Sep 17 00:00:00 2001 From: Juan Fernandez Date: Fri, 28 Oct 2022 11:12:21 -0400 Subject: [PATCH] Feature 7494: Use the MSRC security bulletin artifacts for detecting Win OS vulnerabilities (#7889) Use the MSRC security bulletin artifacts for detecting Win OS vulnerabilities --- ...-7494-use-msrc-bulletins-to-scan-win-vulns | 2 + cmd/fleet/cron.go | 103 ++++---- cmd/fleet/cron_test.go | 78 ------ cmd/fleetctl/vulnerability_data_stream.go | 10 + .../vulnerability_data_stream_test.go | 1 + cmd/msrc/generate.go | 8 +- server/datastore/mysql/hosts.go | 30 +++ server/datastore/mysql/hosts_test.go | 89 +++++++ ...eateOperatingSystemVulnerabilitiesTable.go | 41 ++++ .../mysql/operating_system_vulnerabilities.go | 78 ++++++ .../operating_system_vulnerabilities_test.go | 143 +++++++++++ server/datastore/mysql/schema.sql | 18 +- server/datastore/mysql/software.go | 2 +- server/datastore/mysql/software_test.go | 24 +- server/datastore/mysql/windows_updates.go | 19 ++ .../datastore/mysql/windows_updates_test.go | 40 ++++ server/fleet/datastore.go | 18 +- server/fleet/software.go | 65 +---- server/fleet/vulnerabilities.go | 114 +++++++++ server/mock/datastore_mock.go | 62 ++++- server/service/integration_core_test.go | 4 +- server/vulnerabilities/msrc/analyzer.go | 187 +++++++++++++++ server/vulnerabilities/msrc/analyzer_test.go | 130 ++++++++++ server/vulnerabilities/msrc/io/github.go | 6 +- server/vulnerabilities/msrc/io/github_test.go | 4 +- .../msrc/io/security_bulletin_name.go | 4 +- server/vulnerabilities/msrc/parsed/product.go | 32 ++- .../msrc/parsed/product_test.go | 62 ++++- .../msrc/parsed/security_bulletin.go | 116 ++++++++- .../msrc/parsed/security_bulletin_test.go | 28 +-- server/vulnerabilities/msrc/parser.go | 20 +- server/vulnerabilities/msrc/parser_test.go | 178 +++++++------- server/vulnerabilities/msrc/sync.go | 10 +- server/vulnerabilities/msrc/sync_test.go | 6 +- server/vulnerabilities/nvd/cve.go | 2 +- server/vulnerabilities/nvd/cve_test.go | 12 +- server/vulnerabilities/oval/analyzer.go | 126 +--------- server/vulnerabilities/oval/analyzer_test.go | 129 ---------- .../oval/parsed/dpkg_infotest.go | 7 +- .../oval/parsed/object_info_state.go | 5 +- .../oval/parsed/object_state_simple_value.go | 4 +- .../{oval/parsed => utils}/rpmvercmp.go | 6 +- .../{oval/parsed => utils}/rpmvercmp_test.go | 5 +- server/vulnerabilities/utils/utils.go | 162 +++++++++++++ server/vulnerabilities/utils/utils_test.go | 226 ++++++++++++++++++ server/webhooks/vulnerabilities.go | 3 +- server/webhooks/vulnerabilities_test.go | 23 +- server/worker/jira.go | 4 +- server/worker/zendesk.go | 4 +- 49 files changed, 1830 insertions(+), 620 deletions(-) create mode 100644 changes/feature-7494-use-msrc-bulletins-to-scan-win-vulns delete mode 100644 cmd/fleet/cron_test.go create mode 100644 server/datastore/mysql/migrations/tables/20221027085019_CreateOperatingSystemVulnerabilitiesTable.go create mode 100644 server/datastore/mysql/operating_system_vulnerabilities.go create mode 100644 server/datastore/mysql/operating_system_vulnerabilities_test.go create mode 100644 server/fleet/vulnerabilities.go create mode 100644 server/vulnerabilities/msrc/analyzer.go create mode 100644 server/vulnerabilities/msrc/analyzer_test.go rename server/vulnerabilities/{oval/parsed => utils}/rpmvercmp.go (98%) rename server/vulnerabilities/{oval/parsed => utils}/rpmvercmp_test.go (99%) create mode 100644 server/vulnerabilities/utils/utils.go create mode 100644 server/vulnerabilities/utils/utils_test.go diff --git a/changes/feature-7494-use-msrc-bulletins-to-scan-win-vulns b/changes/feature-7494-use-msrc-bulletins-to-scan-win-vulns new file mode 100644 index 0000000000..44d908bef8 --- /dev/null +++ b/changes/feature-7494-use-msrc-bulletins-to-scan-win-vulns @@ -0,0 +1,2 @@ +* Use the MSRC security bulletins to scan for Windows vulnerabilities. Detected vulnerabilities are +inserted in a new table, 'operating_system_vulnerabilities'. \ No newline at end of file diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index d2be42795a..97650ca766 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -20,8 +20,10 @@ import ( "github.com/fleetdm/fleet/v4/server/policies" "github.com/fleetdm/fleet/v4/server/service/externalsvc" "github.com/fleetdm/fleet/v4/server/service/schedule" + "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc" "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd" "github.com/fleetdm/fleet/v4/server/vulnerabilities/oval" + "github.com/fleetdm/fleet/v4/server/vulnerabilities/utils" "github.com/fleetdm/fleet/v4/server/webhooks" "github.com/fleetdm/fleet/v4/server/worker" "github.com/getsentry/sentry-go" @@ -169,14 +171,31 @@ func scanVulnerabilities( nvdVulns := checkNVDVulnerabilities(ctx, ds, logger, vulnPath, config, vulnAutomationEnabled != "") ovalVulns := checkOvalVulnerabilities(ctx, ds, logger, vulnPath, config, vulnAutomationEnabled != "") - vulns, meta := recentVulns(ctx, ds, logger, nvdVulns, ovalVulns, config.RecentVulnerabilityMaxAge) + checkWinVulnerabilities(ctx, ds, logger, vulnPath, config, vulnAutomationEnabled != "") - if len(vulns) > 0 { + // If no automations enabled, then there is nothing else to do... + if vulnAutomationEnabled == "" { + return nil + } + + vulns := make([]fleet.SoftwareVulnerability, 0, len(nvdVulns)+len(ovalVulns)) + vulns = append(vulns, nvdVulns...) + vulns = append(vulns, ovalVulns...) + + meta, err := ds.ListCVEs(ctx, config.RecentVulnerabilityMaxAge) + if err != nil { + errHandler(ctx, logger, "could not fetch CVE meta", err) + return nil + } + + recentV, matchingMeta := utils.RecentVulns(vulns, meta) + + if len(recentV) > 0 { switch vulnAutomationEnabled { case "webhook": args := webhooks.VulnArgs{ - Vulnerablities: vulns, - Meta: meta, + Vulnerablities: recentV, + Meta: matchingMeta, AppConfig: appConfig, Time: time.Now(), } @@ -201,8 +220,8 @@ func scanVulnerabilities( ctx, ds, kitlog.With(logger, "jira", "vulnerabilities"), - vulns, - meta, + recentV, + matchingMeta, ); err != nil { errHandler(ctx, logger, "queueing vulnerabilities to jira", err) } @@ -213,8 +232,8 @@ func scanVulnerabilities( ctx, ds, kitlog.With(logger, "zendesk", "vulnerabilities"), - vulns, - meta, + recentV, + matchingMeta, ); err != nil { errHandler(ctx, logger, "queueing vulnerabilities to Zendesk", err) } @@ -228,48 +247,52 @@ func scanVulnerabilities( return nil } -// recentVulns filters both the vulnerabilities comming from NVD and OVAL based on 'maxAge' -// (any vulnerability older than 'maxAge' will be excluded). Returns the filtered vulnerabilities -// and their meta data. -func recentVulns( +func checkWinVulnerabilities( ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, - nvdVulns []fleet.SoftwareVulnerability, - ovalVulns []fleet.SoftwareVulnerability, - maxAge time.Duration, -) ([]fleet.SoftwareVulnerability, map[string]fleet.CVEMeta) { - if len(nvdVulns) == 0 && len(ovalVulns) == 0 { - return nil, nil - } + vulnPath string, + config *config.VulnerabilitiesConfig, + collectVulns bool, +) []fleet.OSVulnerability { + var results []fleet.OSVulnerability - meta, err := ds.ListCVEs(ctx, maxAge) + // Get OS + os, err := ds.ListOperatingSystems(ctx) if err != nil { - errHandler(ctx, logger, "could not fetch CVE meta", err) - return nil, nil + errHandler(ctx, logger, "fetching list of operating systems", err) + return nil } - recent := make(map[string]fleet.CVEMeta) - for _, r := range meta { - recent[r.CVE] = r - } - - seen := make(map[string]bool) - var vulns []fleet.SoftwareVulnerability - for _, v := range nvdVulns { - if _, ok := recent[v.CVE]; ok && !seen[v.Key()] { - seen[v.Key()] = true - vulns = append(vulns, v) - } - } - for _, v := range ovalVulns { - if _, ok := recent[v.CVE]; ok && !seen[v.Key()] { - seen[v.Key()] = true - vulns = append(vulns, v) + if !config.DisableDataSync { + // Sync MSRC definitions + client := fleethttp.NewClient() + err = msrc.Sync(ctx, client, vulnPath, os) + if err != nil { + errHandler(ctx, logger, "updating msrc definitions", err) } } - return vulns, recent + // Analyze all Win OS using the synched MSRC artifact. + if !config.DisableWinOSVulnerabilities { + for _, o := range os { + start := time.Now() + r, err := msrc.Analyze(ctx, ds, o, vulnPath, collectVulns) + elapsed := time.Since(start) + level.Debug(logger).Log( + "msg", "msrc-analysis-done", + "os name", o.Name, + "os version", o.Version, + "elapsed", elapsed, + "found new", len(r)) + results = append(results, r...) + if err != nil { + errHandler(ctx, logger, "analyzing hosts for Windows vulnerabilities", err) + } + } + } + + return results } func checkOvalVulnerabilities( diff --git a/cmd/fleet/cron_test.go b/cmd/fleet/cron_test.go deleted file mode 100644 index 16b4ab2fa6..0000000000 --- a/cmd/fleet/cron_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package main - -import ( - "context" - "testing" - "time" - - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/fleetdm/fleet/v4/server/mock" - kitlog "github.com/go-kit/kit/log" - "github.com/stretchr/testify/require" -) - -func TestFilterRecentVulns(t *testing.T) { - t.Run("no NVD nor OVAL vulns", func(t *testing.T) { - ctx := context.Background() - ds := new(mock.Store) - logger := kitlog.NewNopLogger() - - vulns, meta := recentVulns(ctx, ds, logger, nil, nil, 2*time.Hour) - require.Empty(t, vulns) - require.Empty(t, meta) - }) - - t.Run("filters both NVD and OVAL vulns based on max age", func(t *testing.T) { - ctx := context.Background() - ds := new(mock.Store) - logger := kitlog.NewNopLogger() - - dsMeta := []fleet.CVEMeta{ - {CVE: "cve-recent-1"}, - {CVE: "cve-recent-2"}, - {CVE: "cve-recent-3"}, - } - - ds.ListCVEsFunc = func(ctx context.Context, maxAge time.Duration) ([]fleet.CVEMeta, error) { - return dsMeta, nil - } - - ovalVulns := []fleet.SoftwareVulnerability{ - {CVE: "cve-recent-1"}, - {CVE: "cve-recent-2"}, - {CVE: "cve-recent-2"}, - {CVE: "cve-outdated-1"}, - } - - nvdVulns := []fleet.SoftwareVulnerability{ - {CVE: "cve-recent-1"}, - {CVE: "cve-recent-3"}, - {CVE: "cve-outdated-2"}, - {CVE: "cve-outdated-3"}, - } - - maxAge := 30 * 24 * time.Hour - - expected := []string{ - "cve-recent-1", - "cve-recent-2", - "cve-recent-3", - } - - var actual []string - vulns, meta := recentVulns(ctx, ds, logger, nvdVulns, ovalVulns, maxAge) - for _, r := range vulns { - actual = append(actual, r.CVE) - } - - expectedMeta := map[string]fleet.CVEMeta{ - "cve-recent-1": {CVE: "cve-recent-1"}, - "cve-recent-2": {CVE: "cve-recent-2"}, - "cve-recent-3": {CVE: "cve-recent-3"}, - } - - require.Equal(t, len(expected), len(actual)) - require.ElementsMatch(t, expected, actual) - require.Equal(t, expectedMeta, meta) - }) -} diff --git a/cmd/fleetctl/vulnerability_data_stream.go b/cmd/fleetctl/vulnerability_data_stream.go index 1aa268b5ee..b42d4836c9 100644 --- a/cmd/fleetctl/vulnerability_data_stream.go +++ b/cmd/fleetctl/vulnerability_data_stream.go @@ -1,10 +1,12 @@ package main import ( + "context" "errors" "os" "github.com/fleetdm/fleet/v4/pkg/fleethttp" + "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc" "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd" "github.com/fleetdm/fleet/v4/server/vulnerabilities/oval" "github.com/urfave/cli/v2" @@ -84,6 +86,14 @@ Downloads (if needed) the data streams that can be used by the Fleet server to p } log(c, " Done\n") + log(c, "[-] Downloading MSRC artifacts...") + ctx := context.Background() + err = msrc.Sync(ctx, client, dir, nil) + if err != nil { + return err + } + log(c, " Done\n") + log(c, "[+] Data streams successfully downloaded!\n") return nil }, diff --git a/cmd/fleetctl/vulnerability_data_stream_test.go b/cmd/fleetctl/vulnerability_data_stream_test.go index 4894abc213..8207224c37 100644 --- a/cmd/fleetctl/vulnerability_data_stream_test.go +++ b/cmd/fleetctl/vulnerability_data_stream_test.go @@ -23,6 +23,7 @@ func TestVulnerabilityDataStream(t *testing.T) { [-] Downloading EPSS feed... Done [-] Downloading CISA known exploits feed... Done [-] Downloading Oval definitions... Done +[-] Downloading MSRC artifacts... Done [+] Data streams successfully downloaded! ` diff --git a/cmd/msrc/generate.go b/cmd/msrc/generate.go index e09b68658e..f1b68df464 100644 --- a/cmd/msrc/generate.go +++ b/cmd/msrc/generate.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "net/http" @@ -35,11 +36,12 @@ func main() { now := time.Now() httpC := http.DefaultClient - ghAPI := io.NewGitHubClient(httpC, github.NewClient(httpC).Repositories, inPath) + ctx := context.Background() + ghAPI := io.NewGitHubClient(httpC, github.NewClient(httpC).Repositories, wd) msrcAPI := io.NewMSRCClient(httpC, inPath, io.MSRCBaseURL) fmt.Println("Downloading existing bulletins...") - eBulletins, err := ghAPI.Bulletins() + eBulletins, err := ghAPI.Bulletins(ctx) panicif(err) var bulletins []*parsed.SecurityBulletin @@ -151,7 +153,7 @@ func serialize(b *parsed.SecurityBulletin, d time.Time, dir string) error { if err != nil { return err } - fileName := io.FileName(b.ProductName, d, "json") + fileName := io.FileName(b.ProductName, d) filePath := filepath.Join(dir, fileName) return os.WriteFile(filePath, payload, 0o644) diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 14aa14ed45..0a1c1133bf 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -313,6 +313,7 @@ var hostRefs = []string{ "host_display_names", "windows_updates", "host_disks", + "operating_system_vulnerabilities", } func (ds *Datastore) DeleteHost(ctx context.Context, hid uint) error { @@ -2965,6 +2966,35 @@ func (ds *Datastore) CountEnrolledHosts(ctx context.Context) (int, error) { return count, nil } +func (ds *Datastore) HostIDsByOSID( + ctx context.Context, + osID uint, + offset int, + limit int, +) ([]uint, error) { + var ids []uint + + stmt := dialect.From("host_operating_system"). + Select("host_id"). + Where( + goqu.C("os_id").Eq(osID)). + Order(goqu.I("host_id").Desc()). + Offset(uint(offset)). + Limit(uint(limit)) + + sql, args, err := stmt.ToSQL() + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "get host IDs") + } + + if err := sqlx.SelectContext(ctx, ds.reader, &ids, sql, args...); err != nil { + return nil, ctxerr.Wrap(ctx, err, "get host IDs") + } + + return ids, nil +} + +// TODO Refactor this: We should be using the operating system type for this func (ds *Datastore) HostIDsByOSVersion( ctx context.Context, osVersion fleet.OSVersion, diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index 6a07729a27..e64702b12a 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -128,6 +128,7 @@ func TestHosts(t *testing.T) { {"CountHostsNotResponding", testCountHostsNotResponding}, {"FailingPoliciesCount", testFailingPoliciesCount}, {"SetOrUpdateHostDisksSpace", testHostsSetOrUpdateHostDisksSpace}, + {"HostIDsByOSID", testHostIDsByOSID}, {"TestHostDisplayName", testHostDisplayName}, } for _, c := range cases { @@ -4895,6 +4896,13 @@ func testHostsDeleteHosts(t *testing.T, ds *Datastore) { err = ds.SetOrUpdateHostOrbitInfo(context.Background(), host.ID, "1.1.0") require.NoError(t, err) + // Operating system vulnerabilities + _, err = ds.writer.Exec( + `INSERT INTO operating_system_vulnerabilities(host_id,operating_system_id,cve) VALUES (?,?,?)`, + host.ID, 1, "cve-1", + ) + require.NoError(t, err) + // Check there's an entry for the host in all the associated tables. for _, hostRef := range hostRefs { var ok bool @@ -5342,3 +5350,84 @@ func testHostDisplayName(t *testing.T, ds *Datastore) { assert.Equal(t, expect[i], h.DisplayName()) } } + +func testHostIDsByOSID(t *testing.T, ds *Datastore) { + ctx := context.Background() + + t.Run("no OS", func(t *testing.T) { + actual, err := ds.HostIDsByOSID(ctx, 1, 0, 100) + require.NoError(t, err) + require.Empty(t, actual) + }) + + t.Run("returns empty if no more pages", func(t *testing.T) { + for i := 1; i <= 510; i++ { + os := fleet.OperatingSystem{ + Name: "Microsoft Windows 11 Enterprise Evaluation II", + Version: "21H2", + Arch: "64-bit", + KernelVersion: "10.0.22000.795", + Platform: "windows", + } + + require.NoError(t, ds.UpdateHostOperatingSystem(ctx, uint(i+100), os)) + } + + storedOS, err := ds.ListOperatingSystems(ctx) + require.NoError(t, err) + for _, sOS := range storedOS { + if sOS.Name == "Microsoft Windows 11 Enterprise Evaluation II" { + + actual, err := ds.HostIDsByOSID(ctx, sOS.ID, 0, 500) + require.NoError(t, err) + require.Len(t, actual, 500) + + actual, err = ds.HostIDsByOSID(ctx, sOS.ID, 500, 500) + require.NoError(t, err) + require.Len(t, actual, 10) + + actual, err = ds.HostIDsByOSID(ctx, sOS.ID, 510, 500) + require.NoError(t, err) + require.Empty(t, actual) + break + } + } + }) + + t.Run("returns matching entries", func(t *testing.T) { + os := []fleet.OperatingSystem{ + { + Name: "Microsoft Windows 11 Enterprise Evaluation", + Version: "21H2", + Arch: "64-bit", + KernelVersion: "10.0.22000.795", + Platform: "windows", + }, + { + Name: "macOS", + Version: "12.3.1", + Arch: "x86_64", + KernelVersion: "21.4.0", + Platform: "darwin", + }, + } + + require.NoError(t, ds.UpdateHostOperatingSystem(ctx, 1, os[0])) + require.NoError(t, ds.UpdateHostOperatingSystem(ctx, 2, os[1])) + + storedOS, err := ds.ListOperatingSystems(ctx) + require.NoError(t, err) + + for _, sOS := range storedOS { + actual, err := ds.HostIDsByOSID(ctx, sOS.ID, 0, 100) + require.NoError(t, err) + if sOS.Name == "Microsoft Windows 11 Enterprise Evaluation" { + require.Equal(t, []uint{1}, actual) + } + + if sOS.Name == "macOS" { + require.Equal(t, []uint{2}, actual) + } + } + }) +} diff --git a/server/datastore/mysql/migrations/tables/20221027085019_CreateOperatingSystemVulnerabilitiesTable.go b/server/datastore/mysql/migrations/tables/20221027085019_CreateOperatingSystemVulnerabilitiesTable.go new file mode 100644 index 0000000000..1d02d8e0af --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20221027085019_CreateOperatingSystemVulnerabilitiesTable.go @@ -0,0 +1,41 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20221027085019, Down_20221027085019) +} + +func Up_20221027085019(tx *sql.Tx) error { + logger.Info.Println("Creating table operating_system_vulnerabilities...") + + _, err := tx.Exec(` + CREATE TABLE operating_system_vulnerabilities + ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + host_id INT UNSIGNED NOT NULL, + operating_system_id INT UNSIGNED NOT NULL, + cve VARCHAR(255) NOT NULL, + source SMALLINT DEFAULT 0, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + + UNIQUE KEY idx_operating_system_vulnerabilities_unq_cve (host_id, cve), + INDEX idx_operating_system_vulnerabilities_operating_system_id_cve (operating_system_id, cve) + ) + `) + if err != nil { + return errors.Wrapf(err, "operating_system_vulnerabilities") + } + + logger.Info.Println("Done creating table operating_system_vulnerabilities...") + + return nil +} + +func Down_20221027085019(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/operating_system_vulnerabilities.go b/server/datastore/mysql/operating_system_vulnerabilities.go new file mode 100644 index 0000000000..57752fb90d --- /dev/null +++ b/server/datastore/mysql/operating_system_vulnerabilities.go @@ -0,0 +1,78 @@ +package mysql + +import ( + "context" + "fmt" + "strings" + + "github.com/doug-martin/goqu/v9" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/jmoiron/sqlx" +) + +func (ds *Datastore) ListOSVulnerabilities(ctx context.Context, hostIDs []uint) ([]fleet.OSVulnerability, error) { + r := []fleet.OSVulnerability{} + + stmt := dialect. + From(goqu.T("operating_system_vulnerabilities")). + Select( + goqu.I("host_id"), + goqu.I("operating_system_id"), + goqu.I("cve"), + ). + Where(goqu.C("host_id").In(hostIDs)) + + sql, args, err := stmt.ToSQL() + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "error generating SQL statement") + } + + if err := sqlx.SelectContext(ctx, ds.reader, &r, sql, args...); err != nil { + return nil, ctxerr.Wrap(ctx, err, "error executing SQL statement") + } + + return r, nil +} + +func (ds *Datastore) InsertOSVulnerabilities(ctx context.Context, vulnerabilities []fleet.OSVulnerability, source fleet.VulnerabilitySource) (int64, error) { + var args []interface{} + + if len(vulnerabilities) == 0 { + return 0, nil + } + + values := strings.TrimSuffix(strings.Repeat("(?,?,?,?),", len(vulnerabilities)), ",") + sql := fmt.Sprintf(`INSERT IGNORE INTO operating_system_vulnerabilities (host_id, operating_system_id, cve, source) VALUES %s`, values) + + for _, v := range vulnerabilities { + args = append(args, v.HostID, v.OSID, v.CVE, source) + } + res, err := ds.writer.ExecContext(ctx, sql, args...) + if err != nil { + return 0, ctxerr.Wrap(ctx, err, "insert operating system vulnerabilities") + } + count, _ := res.RowsAffected() + + return count, nil +} + +func (ds *Datastore) DeleteOSVulnerabilities(ctx context.Context, vulnerabilities []fleet.OSVulnerability) error { + if len(vulnerabilities) == 0 { + return nil + } + + sql := fmt.Sprintf( + `DELETE FROM operating_system_vulnerabilities WHERE (host_id, cve) IN (%s)`, + strings.TrimSuffix(strings.Repeat("(?,?),", len(vulnerabilities)), ","), + ) + + var args []interface{} + for _, v := range vulnerabilities { + args = append(args, v.HostID, v.CVE) + } + if _, err := ds.writer.ExecContext(ctx, sql, args...); err != nil { + return ctxerr.Wrapf(ctx, err, "deleting operating system vulnerabilities") + } + return nil +} diff --git a/server/datastore/mysql/operating_system_vulnerabilities_test.go b/server/datastore/mysql/operating_system_vulnerabilities_test.go new file mode 100644 index 0000000000..263b451fd2 --- /dev/null +++ b/server/datastore/mysql/operating_system_vulnerabilities_test.go @@ -0,0 +1,143 @@ +package mysql + +import ( + "context" + "testing" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" +) + +func TestOperatingSystemVulnerabilities(t *testing.T) { + ds := CreateMySQLDS(t) + + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"ListOSVulnerabilitiesEmpty", testListOSVulnerabilitiesEmpty}, + {"ListOSVulnerabilities", testListOSVulnerabilities}, + {"InsertOSVulnerabilities", testInsertOSVulnerabilities}, + {"DeleteOSVulnerabilitiesEmpty", testDeleteOSVulnerabilitiesEmpty}, + {"DeleteOSVulnerabilities", testDeleteOSVulnerabilities}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func testListOSVulnerabilitiesEmpty(t *testing.T, ds *Datastore) { + ctx := context.Background() + actual, err := ds.ListOSVulnerabilities(ctx, []uint{4}) + require.NoError(t, err) + require.Empty(t, actual) +} + +func testListOSVulnerabilities(t *testing.T, ds *Datastore) { + ctx := context.Background() + + vulns := []fleet.OSVulnerability{ + {HostID: 1, CVE: "cve-1", OSID: 1}, + {HostID: 1, CVE: "cve-3", OSID: 1}, + {HostID: 2, CVE: "cve-2", OSID: 1}, + } + + for _, v := range vulns { + _, err := ds.writer.Exec( + `INSERT INTO operating_system_vulnerabilities(host_id,operating_system_id,cve) VALUES (?,?,?)`, + v.HostID, v.OSID, v.CVE, + ) + require.NoError(t, err) + } + + t.Run("none matching", func(t *testing.T) { + actual, err := ds.ListOSVulnerabilities(ctx, []uint{3}) + require.NoError(t, err) + require.Empty(t, actual) + }) + + t.Run("returns matching", func(t *testing.T) { + expected := []fleet.OSVulnerability{ + {HostID: 1, CVE: "cve-1", OSID: 1}, + {HostID: 1, CVE: "cve-3", OSID: 1}, + } + + actual, err := ds.ListOSVulnerabilities(ctx, []uint{1}) + require.NoError(t, err) + require.ElementsMatch(t, expected, actual) + }) +} + +func testInsertOSVulnerabilities(t *testing.T, ds *Datastore) { + ctx := context.Background() + + vulns := []fleet.OSVulnerability{ + {HostID: 1, CVE: "cve-1", OSID: 1}, + {HostID: 1, CVE: "cve-1", OSID: 1}, + {HostID: 1, CVE: "cve-3", OSID: 1}, + {HostID: 2, CVE: "cve-2", OSID: 1}, + } + + c, err := ds.InsertOSVulnerabilities(ctx, vulns, fleet.MSRCSource) + require.NoError(t, err) + require.Equal(t, int64(3), c) + + expected := []fleet.OSVulnerability{ + {HostID: 1, CVE: "cve-1", OSID: 1}, + {HostID: 1, CVE: "cve-3", OSID: 1}, + } + + actual, err := ds.ListOSVulnerabilities(ctx, []uint{1}) + require.NoError(t, err) + require.ElementsMatch(t, expected, actual) +} + +func testDeleteOSVulnerabilitiesEmpty(t *testing.T, ds *Datastore) { + ctx := context.Background() + + vulns := []fleet.OSVulnerability{ + {HostID: 1, CVE: "cve-1", OSID: 1}, + {HostID: 1, CVE: "cve-1", OSID: 1}, + {HostID: 1, CVE: "cve-3", OSID: 1}, + {HostID: 2, CVE: "cve-2", OSID: 1}, + } + + err := ds.DeleteOSVulnerabilities(ctx, vulns) + require.NoError(t, err) +} + +func testDeleteOSVulnerabilities(t *testing.T, ds *Datastore) { + ctx := context.Background() + + vulns := []fleet.OSVulnerability{ + {HostID: 1, CVE: "cve-1", OSID: 1}, + {HostID: 1, CVE: "cve-1", OSID: 1}, + {HostID: 1, CVE: "cve-3", OSID: 1}, + {HostID: 2, CVE: "cve-2", OSID: 1}, + } + + c, err := ds.InsertOSVulnerabilities(ctx, vulns, fleet.MSRCSource) + require.NoError(t, err) + require.Equal(t, int64(3), c) + + toDelete := []fleet.OSVulnerability{ + {HostID: 2, CVE: "cve-2", OSID: 1}, + } + + err = ds.DeleteOSVulnerabilities(ctx, toDelete) + require.NoError(t, err) + + actual, err := ds.ListOSVulnerabilities(ctx, []uint{1}) + require.NoError(t, err) + require.ElementsMatch(t, []fleet.OSVulnerability{ + {HostID: 1, CVE: "cve-1", OSID: 1}, + {HostID: 1, CVE: "cve-3", OSID: 1}, + }, actual) + + actual, err = ds.ListOSVulnerabilities(ctx, []uint{2}) + require.NoError(t, err) + require.Empty(t, actual) +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 60627179cd..b250ff87a9 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -451,9 +451,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=154 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=155 DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `mobile_device_management_solutions` ( @@ -673,6 +673,20 @@ CREATE TABLE `network_interfaces` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; +CREATE TABLE `operating_system_vulnerabilities` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `host_id` int(10) unsigned NOT NULL, + `operating_system_id` int(10) unsigned NOT NULL, + `cve` varchar(255) NOT NULL, + `source` smallint(6) DEFAULT '0', + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `idx_operating_system_vulnerabilities_unq_cve` (`host_id`,`cve`), + KEY `idx_operating_system_vulnerabilities_operating_system_id_cve` (`operating_system_id`,`cve`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +/*!40101 SET character_set_client = @saved_cs_client */; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; CREATE TABLE `operating_systems` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `name` varchar(255) NOT NULL, diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index de7065643b..59abb75870 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -1037,7 +1037,7 @@ ON DUPLICATE KEY UPDATE return nil } -func (ds *Datastore) InsertVulnerabilities( +func (ds *Datastore) InsertSoftwareVulnerabilities( ctx context.Context, vulns []fleet.SoftwareVulnerability, source fleet.VulnerabilitySource, diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index cf91ec5fd9..1cba1441c1 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -39,7 +39,7 @@ func TestSoftware(t *testing.T) { {"UpdateHostSoftware", testUpdateHostSoftware}, {"ListSoftwareByHostIDShort", testListSoftwareByHostIDShort}, {"ListSoftwareVulnerabilities", testListSoftwareVulnerabilities}, - {"InsertVulnerabilities", testInsertVulnerabilities}, + {"InsertSoftwareVulnerabilities", testInsertSoftwareVulnerabilities}, {"ListCVEs", testListCVEs}, {"ListSoftwareForVulnDetection", testListSoftwareForVulnDetection}, {"SoftwareByID", testSoftwareByID}, @@ -258,7 +258,7 @@ func testSoftwareLoadVulnerabilities(t *testing.T, ds *Datastore) { {SoftwareID: host.Software[0].ID, CVE: "CVE-2022-0001"}, {SoftwareID: host.Software[0].ID, CVE: "CVE-2022-0002"}, } - _, err := ds.InsertVulnerabilities(context.Background(), vulns, fleet.NVDSource) + _, err := ds.InsertSoftwareVulnerabilities(context.Background(), vulns, fleet.NVDSource) require.NoError(t, err) require.NoError(t, ds.LoadHostSoftware(context.Background(), host, false)) @@ -514,7 +514,7 @@ func testSoftwareList(t *testing.T, ds *Datastore) { {SoftwareID: host3.Software[0].ID, CVE: "CVE-2022-0003"}, } - _, err := ds.InsertVulnerabilities(context.Background(), vulns, fleet.NVDSource) + _, err := ds.InsertSoftwareVulnerabilities(context.Background(), vulns, fleet.NVDSource) require.NoError(t, err) cveMeta := []fleet.CVEMeta{ @@ -1120,7 +1120,7 @@ func insertVulnSoftwareForTest(t *testing.T, ds *Datastore) { }) chrome3 := host2.Software[2] - n, err := ds.InsertVulnerabilities(context.Background(), []fleet.SoftwareVulnerability{ + n, err := ds.InsertSoftwareVulnerabilities(context.Background(), []fleet.SoftwareVulnerability{ { SoftwareID: chrome3.ID, CVE: "CVE-2022-0001", @@ -1131,7 +1131,7 @@ func insertVulnSoftwareForTest(t *testing.T, ds *Datastore) { require.Equal(t, 1, int(n)) barRpm := host2.Software[0] - n, err = ds.InsertVulnerabilities(context.Background(), + n, err = ds.InsertSoftwareVulnerabilities(context.Background(), []fleet.SoftwareVulnerability{ { SoftwareID: barRpm.ID, @@ -1457,7 +1457,7 @@ func testListSoftwareVulnerabilities(t *testing.T, ds *Datastore) { } } - n, err := ds.InsertVulnerabilities(ctx, vulns, fleet.NVDSource) + n, err := ds.InsertSoftwareVulnerabilities(ctx, vulns, fleet.NVDSource) require.NoError(t, err) require.Equal(t, int(n), 2) @@ -1477,11 +1477,11 @@ func testListSoftwareVulnerabilities(t *testing.T, ds *Datastore) { } } -func testInsertVulnerabilities(t *testing.T, ds *Datastore) { +func testInsertSoftwareVulnerabilities(t *testing.T, ds *Datastore) { ctx := context.Background() t.Run("no vulnerabilities to insert", func(t *testing.T) { - r, err := ds.InsertVulnerabilities(ctx, nil, fleet.UbuntuOVALSource) + r, err := ds.InsertSoftwareVulnerabilities(ctx, nil, fleet.UbuntuOVALSource) require.Zero(t, r) require.NoError(t, err) }) @@ -1506,7 +1506,7 @@ func testInsertVulnerabilities(t *testing.T, ds *Datastore) { }) } - n, err := ds.InsertVulnerabilities(ctx, vulns, fleet.UbuntuOVALSource) + n, err := ds.InsertSoftwareVulnerabilities(ctx, vulns, fleet.UbuntuOVALSource) require.NoError(t, err) require.Equal(t, 1, int(n)) @@ -1538,11 +1538,11 @@ func testInsertVulnerabilities(t *testing.T, ds *Datastore) { }) } - n, err := ds.InsertVulnerabilities(ctx, vulns, fleet.UbuntuOVALSource) + n, err := ds.InsertSoftwareVulnerabilities(ctx, vulns, fleet.UbuntuOVALSource) require.NoError(t, err) require.Equal(t, 1, int(n)) - n, err = ds.InsertVulnerabilities(ctx, vulns, fleet.UbuntuOVALSource) + n, err = ds.InsertSoftwareVulnerabilities(ctx, vulns, fleet.UbuntuOVALSource) require.NoError(t, err) require.Equal(t, 0, int(n)) @@ -1660,7 +1660,7 @@ func testSoftwareByID(t *testing.T, ds *Datastore) { CVE: fmt.Sprintf("cve-%d", i), }) } - n, err := ds.InsertVulnerabilities(ctx, vulns, fleet.UbuntuOVALSource) + n, err := ds.InsertSoftwareVulnerabilities(ctx, vulns, fleet.UbuntuOVALSource) require.NoError(t, err) require.Equal(t, 4, int(n)) diff --git a/server/datastore/mysql/windows_updates.go b/server/datastore/mysql/windows_updates.go index 5ec71e1e98..c83d8a9833 100644 --- a/server/datastore/mysql/windows_updates.go +++ b/server/datastore/mysql/windows_updates.go @@ -11,6 +11,25 @@ import ( "github.com/jmoiron/sqlx" ) +func (ds *Datastore) ListWindowsUpdatesByHostID( + ctx context.Context, + hostID uint, +) ([]fleet.WindowsUpdate, error) { + stmt := ` + SELECT kb_id, date_epoch + FROM windows_updates wu + WHERE host_id = ? + ORDER BY date_epoch + ` + updates := []fleet.WindowsUpdate{} + + if err := sqlx.SelectContext(ctx, ds.reader, &updates, stmt, hostID); err != nil { + return nil, ctxerr.Wrap(ctx, err, "list windows updates") + } + + return updates, nil +} + // InsertWindowsUpdates inserts one or more windows updates for the given host. func (ds *Datastore) InsertWindowsUpdates(ctx context.Context, hostID uint, updates []fleet.WindowsUpdate) error { if len(updates) == 0 { diff --git a/server/datastore/mysql/windows_updates_test.go b/server/datastore/mysql/windows_updates_test.go index a4d5329bba..d55c3f50f5 100644 --- a/server/datastore/mysql/windows_updates_test.go +++ b/server/datastore/mysql/windows_updates_test.go @@ -18,6 +18,7 @@ func TestWindowsUpdates(t *testing.T) { fn func(t *testing.T, ds *Datastore) }{ {"InsertWindowsUpdates", testInsertWindowsUpdates}, + {"ListWindowsUpdatesByHostID", testListWindowsUpdatesByHostID}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -27,6 +28,45 @@ func TestWindowsUpdates(t *testing.T) { } } +func testListWindowsUpdatesByHostID(t *testing.T, ds *Datastore) { + ctx := context.Background() + now := uint(time.Now().Unix()) + + t.Run("with no stored updates", func(t *testing.T) { + actual, err := ds.ListWindowsUpdatesByHostID(ctx, 1) + require.NoError(t, err) + require.Empty(t, actual) + }) + + t.Run("none matching", func(t *testing.T) { + updates := []fleet.WindowsUpdate{ + {KBID: 1, DateEpoch: now}, + {KBID: 2, DateEpoch: now + 1}, + } + + err := ds.InsertWindowsUpdates(ctx, 1, updates) + require.NoError(t, err) + + actual, err := ds.ListWindowsUpdatesByHostID(ctx, 2) + require.NoError(t, err) + require.Empty(t, actual) + }) + + t.Run("returns matching", func(t *testing.T) { + expected := []fleet.WindowsUpdate{ + {KBID: 1, DateEpoch: now}, + {KBID: 2, DateEpoch: now + 1}, + } + + err := ds.InsertWindowsUpdates(ctx, 1, expected) + require.NoError(t, err) + + actual, err := ds.ListWindowsUpdatesByHostID(ctx, 1) + require.NoError(t, err) + require.ElementsMatch(t, expected, actual) + }) +} + func testInsertWindowsUpdates(t *testing.T, ds *Datastore) { ctx := context.Background() now := uint(time.Now().Unix()) diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 0a4af285a6..c081878939 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -209,6 +209,11 @@ type Datastore interface { GenerateHostStatusStatistics(ctx context.Context, filter TeamFilter, now time.Time, platform *string, lowDiskSpace *int) (*HostSummary, error) // HostIDsByName Retrieve the IDs associated with the given hostnames HostIDsByName(ctx context.Context, filter TeamFilter, hostnames []string) ([]uint, error) + + // HostIDsByOSID retrieves the IDs of all host for the given OS ID + HostIDsByOSID(ctx context.Context, osID uint, offset int, limit int) ([]uint, error) + + // TODO JUAN: Refactor this to use the Operating System type instead. // HostIDsByOSVersion retrieves the IDs of all host matching osVersion HostIDsByOSVersion(ctx context.Context, osVersion OSVersion, offset int, limit int) ([]uint, error) // HostByIdentifier returns one host matching the provided identifier. Possible matches can be on @@ -389,9 +394,9 @@ type Datastore interface { AllSoftwareWithoutCPEIterator(ctx context.Context, excludedPlatforms []string) (SoftwareIterator, error) AddCPEForSoftware(ctx context.Context, software Software, cpe string) error ListSoftwareCPEs(ctx context.Context) ([]SoftwareCPE, error) - // InsertVulnerabilities inserts the given vulnerabilities in the datastore, returns the number + // InsertSoftwareVulnerabilities inserts the given vulnerabilities in the datastore, returns the number // of rows inserted. If a vulnerability already exists in the datastore, then it will be ignored. - InsertVulnerabilities(ctx context.Context, vulns []SoftwareVulnerability, source VulnerabilitySource) (int64, error) + InsertSoftwareVulnerabilities(ctx context.Context, vulns []SoftwareVulnerability, source VulnerabilitySource) (int64, error) SoftwareByID(ctx context.Context, id uint, includeCVEScores bool) (*Software, error) // ListSoftwareByHostIDShort lists software by host ID, but does not include CPEs or vulnerabilites. // It is meant to be used when only minimal software fields are required eg when updating host software. @@ -642,9 +647,16 @@ type Datastore interface { InnoDBStatus(ctx context.Context) (string, error) ProcessList(ctx context.Context) ([]MySQLProcess, error) - // Windows Update History + // WindowsUpdates Store + ListWindowsUpdatesByHostID(ctx context.Context, hostID uint) ([]WindowsUpdate, error) InsertWindowsUpdates(ctx context.Context, hostID uint, updates []WindowsUpdate) error + /////////////////////////////////////////////////////////////////////////////// + // OperatingSystemVulnerabilities Store + ListOSVulnerabilities(ctx context.Context, hostID []uint) ([]OSVulnerability, error) + InsertOSVulnerabilities(ctx context.Context, vulnerabilities []OSVulnerability, source VulnerabilitySource) (int64, error) + DeleteOSVulnerabilities(ctx context.Context, vulnerabilities []OSVulnerability) error + /////////////////////////////////////////////////////////////////////////////// // Apple MDM diff --git a/server/fleet/software.go b/server/fleet/software.go index 688075394a..8f5841676a 100644 --- a/server/fleet/software.go +++ b/server/fleet/software.go @@ -1,43 +1,15 @@ package fleet import ( - "fmt" "time" ) -type CVE struct { - CVE string `json:"cve" db:"cve"` - DetailsLink string `json:"details_link" db:"-"` - // These are double pointers so that we can omit them AND return nulls when needed. - // 1. omitted when using the free tier - // 2. null when using the premium tier, but there is no value available. This may be due to an issue with syncing cve scores. - // 3. non-null when using the premium tier, and value is available. - CVSSScore **float64 `json:"cvss_score,omitempty" db:"cvss_score"` - EPSSProbability **float64 `json:"epss_probability,omitempty" db:"epss_probability"` - CISAKnownExploit **bool `json:"cisa_known_exploit,omitempty" db:"cisa_known_exploit"` -} - -type CVEMeta struct { - CVE string `db:"cve"` - // CVSSScore is the Common Vulnerability Scoring System (CVSS) base score v3. The base score ranges from 0 - 10 and - // takes into account several different metrics. - // See https://nvd.nist.gov/vuln-metrics/cvss. - CVSSScore *float64 `db:"cvss_score"` - // EPSSProbability is the Exploit Prediction Scoring System (EPSS) score. It is the probability - // that a software vulnerability will be exploited in the next 30 days. - // See https://www.first.org/epss/. - EPSSProbability *float64 `db:"epss_probability"` - // CISAKnownExploit is whether the the software vulnerability is a known exploit according to CISA. - // See https://www.cisa.gov/known-exploited-vulnerabilities. - CISAKnownExploit *bool `db:"cisa_known_exploit"` - // Published is when the cve was published according to NIST.score - Published *time.Time `db:"published"` -} - // Must be kept in sync with the vendor column definition. const SoftwareVendorMaxLength = 114 const SoftwareVendorMaxLengthFmt = "%.111s..." +type Vulnerabilities []CVE + // Software is a named and versioned piece of software installed on a device. type Software struct { ID uint `json:"id" db:"id"` @@ -96,8 +68,6 @@ func (s *AuthzSoftwareInventory) AuthzType() string { return "software_inventory" } -type Vulnerabilities []CVE - // HostSoftware is the set of software installed on a specific host type HostSoftware struct { // Software is the software information. @@ -125,34 +95,3 @@ type SoftwareListOptions struct { // a count of hosts > 0. WithHostCounts bool } - -// SoftwareCPE represents an entry in the `software_cpe` table -type SoftwareCPE struct { - ID uint `db:"id"` - SoftwareID uint `db:"software_id"` - CPE string `db:"cpe"` -} - -// SoftwareVulnerability identifies a vulnerability on a specific software. -type SoftwareVulnerability struct { - SoftwareID uint `db:"software_id"` - CVE string `db:"cve"` -} - -// String implements fmt.Stringer. -func (sv SoftwareVulnerability) String() string { - return fmt.Sprintf("{%d,%s}", sv.SoftwareID, sv.CVE) -} - -// Key returns a string representation of the SoftwareVulnerability -func (sv *SoftwareVulnerability) Key() string { - return fmt.Sprintf("%d:%s", sv.SoftwareID, sv.CVE) -} - -type VulnerabilitySource int - -const ( - NVDSource VulnerabilitySource = iota - UbuntuOVALSource - RHELOVALSource -) diff --git a/server/fleet/vulnerabilities.go b/server/fleet/vulnerabilities.go new file mode 100644 index 0000000000..dcbaff9d2c --- /dev/null +++ b/server/fleet/vulnerabilities.go @@ -0,0 +1,114 @@ +package fleet + +import ( + "fmt" + "time" +) + +type CVE struct { + CVE string `json:"cve" db:"cve"` + DetailsLink string `json:"details_link" db:"-"` + // These are double pointers so that we can omit them AND return nulls when needed. + // 1. omitted when using the free tier + // 2. null when using the premium tier, but there is no value available. This may be due to an issue with syncing cve scores. + // 3. non-null when using the premium tier, and value is available. + CVSSScore **float64 `json:"cvss_score,omitempty" db:"cvss_score"` + EPSSProbability **float64 `json:"epss_probability,omitempty" db:"epss_probability"` + CISAKnownExploit **bool `json:"cisa_known_exploit,omitempty" db:"cisa_known_exploit"` +} + +type CVEMeta struct { + CVE string `db:"cve"` + // CVSSScore is the Common Vulnerability Scoring System (CVSS) base score v3. The base score ranges from 0 - 10 and + // takes into account several different metrics. + // See https://nvd.nist.gov/vuln-metrics/cvss. + CVSSScore *float64 `db:"cvss_score"` + // EPSSProbability is the Exploit Prediction Scoring System (EPSS) score. It is the probability + // that a software vulnerability will be exploited in the next 30 days. + // See https://www.first.org/epss/. + EPSSProbability *float64 `db:"epss_probability"` + // CISAKnownExploit is whether the the software vulnerability is a known exploit according to CISA. + // See https://www.cisa.gov/known-exploited-vulnerabilities. + CISAKnownExploit *bool `db:"cisa_known_exploit"` + // Published is when the cve was published according to NIST.score + Published *time.Time `db:"published"` +} + +// SoftwareCPE represents an entry in the `software_cpe` table. +type SoftwareCPE struct { + ID uint `db:"id"` + SoftwareID uint `db:"software_id"` + CPE string `db:"cpe"` +} + +// SoftwareVulnerability is a vulnerability on a software. +// Represents an entry in the `software_cve` table. +type SoftwareVulnerability struct { + SoftwareID uint `db:"software_id"` + CVE string `db:"cve"` +} + +// String implements fmt.Stringer. +func (sv SoftwareVulnerability) String() string { + return fmt.Sprintf("{%d,%s}", sv.SoftwareID, sv.CVE) +} + +// Key returns a string representation of the software vulnerability. +// If we have a list of software vulnerabilities, the Key can be used +// as a discrimator for unique entries. +func (sv SoftwareVulnerability) Key() string { + return fmt.Sprintf("software:%d:%s", sv.SoftwareID, sv.CVE) +} + +func (sv SoftwareVulnerability) GetCVE() string { + return sv.CVE +} + +func (sv SoftwareVulnerability) Affected() uint { + return sv.SoftwareID +} + +// OSVulnerability is a vulnerability on a OS. +// Represents an entry in the `os_vulnerabilities` table. +type OSVulnerability struct { + OSID uint `db:"operating_system_id"` + HostID uint `db:"host_id"` + CVE string `db:"cve"` +} + +// String implements fmt.Stringer. +func (ov OSVulnerability) String() string { + return fmt.Sprintf("{%d,%d,%s}", ov.OSID, ov.HostID, ov.CVE) +} + +// Key returns a string representation of the os vulnerability. +// If we have a list of os vulnerabilities, the Key can be used +// as a discrimator for unique entries. +func (ov OSVulnerability) Key() string { + return fmt.Sprintf("os:%d:%d:%s", ov.OSID, ov.HostID, ov.CVE) +} + +func (ov OSVulnerability) GetCVE() string { + return ov.CVE +} + +func (ov OSVulnerability) Affected() uint { + return ov.HostID +} + +// Represents a vulnerability, e.g. an OS or a Software vulnerability. +type Vulnerability interface { + OSVulnerability | SoftwareVulnerability + GetCVE() string + Affected() uint + Key() string +} + +type VulnerabilitySource int + +const ( + NVDSource VulnerabilitySource = iota + UbuntuOVALSource + RHELOVALSource + MSRCSource +) diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index d65e77593a..89e074fa24 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -165,6 +165,8 @@ type GenerateHostStatusStatisticsFunc func(ctx context.Context, filter fleet.Tea type HostIDsByNameFunc func(ctx context.Context, filter fleet.TeamFilter, hostnames []string) ([]uint, error) +type HostIDsByOSIDFunc func(ctx context.Context, osID uint, offset int, limit int) ([]uint, error) + type HostIDsByOSVersionFunc func(ctx context.Context, osVersion fleet.OSVersion, offset int, limit int) ([]uint, error) type HostByIdentifierFunc func(ctx context.Context, identifier string) (*fleet.Host, error) @@ -311,7 +313,7 @@ type AddCPEForSoftwareFunc func(ctx context.Context, software fleet.Software, cp type ListSoftwareCPEsFunc func(ctx context.Context) ([]fleet.SoftwareCPE, error) -type InsertVulnerabilitiesFunc func(ctx context.Context, vulns []fleet.SoftwareVulnerability, source fleet.VulnerabilitySource) (int64, error) +type InsertSoftwareVulnerabilitiesFunc func(ctx context.Context, vulns []fleet.SoftwareVulnerability, source fleet.VulnerabilitySource) (int64, error) type SoftwareByIDFunc func(ctx context.Context, id uint, includeCVEScores bool) (*fleet.Software, error) @@ -463,8 +465,16 @@ type InnoDBStatusFunc func(ctx context.Context) (string, error) type ProcessListFunc func(ctx context.Context) ([]fleet.MySQLProcess, error) +type ListWindowsUpdatesByHostIDFunc func(ctx context.Context, hostID uint) ([]fleet.WindowsUpdate, error) + type InsertWindowsUpdatesFunc func(ctx context.Context, hostID uint, updates []fleet.WindowsUpdate) error +type ListOSVulnerabilitiesFunc func(ctx context.Context, hostID []uint) ([]fleet.OSVulnerability, error) + +type InsertOSVulnerabilitiesFunc func(ctx context.Context, vulnerabilities []fleet.OSVulnerability, source fleet.VulnerabilitySource) (int64, error) + +type DeleteOSVulnerabilitiesFunc func(ctx context.Context, vulnerabilities []fleet.OSVulnerability) error + type NewMDMAppleEnrollmentProfileFunc func(ctx context.Context, enrollmentPayload fleet.MDMAppleEnrollmentProfilePayload) (*fleet.MDMAppleEnrollmentProfile, error) type GetMDMAppleEnrollmentProfileByTokenFunc func(ctx context.Context, token string) (*fleet.MDMAppleEnrollmentProfile, error) @@ -716,6 +726,9 @@ type DataStore struct { HostIDsByNameFunc HostIDsByNameFunc HostIDsByNameFuncInvoked bool + HostIDsByOSIDFunc HostIDsByOSIDFunc + HostIDsByOSIDFuncInvoked bool + HostIDsByOSVersionFunc HostIDsByOSVersionFunc HostIDsByOSVersionFuncInvoked bool @@ -935,8 +948,8 @@ type DataStore struct { ListSoftwareCPEsFunc ListSoftwareCPEsFunc ListSoftwareCPEsFuncInvoked bool - InsertVulnerabilitiesFunc InsertVulnerabilitiesFunc - InsertVulnerabilitiesFuncInvoked bool + InsertSoftwareVulnerabilitiesFunc InsertSoftwareVulnerabilitiesFunc + InsertSoftwareVulnerabilitiesFuncInvoked bool SoftwareByIDFunc SoftwareByIDFunc SoftwareByIDFuncInvoked bool @@ -1163,9 +1176,21 @@ type DataStore struct { ProcessListFunc ProcessListFunc ProcessListFuncInvoked bool + ListWindowsUpdatesByHostIDFunc ListWindowsUpdatesByHostIDFunc + ListWindowsUpdatesByHostIDFuncInvoked bool + InsertWindowsUpdatesFunc InsertWindowsUpdatesFunc InsertWindowsUpdatesFuncInvoked bool + ListOSVulnerabilitiesFunc ListOSVulnerabilitiesFunc + ListOSVulnerabilitiesFuncInvoked bool + + InsertOSVulnerabilitiesFunc InsertOSVulnerabilitiesFunc + InsertOSVulnerabilitiesFuncInvoked bool + + DeleteOSVulnerabilitiesFunc DeleteOSVulnerabilitiesFunc + DeleteOSVulnerabilitiesFuncInvoked bool + NewMDMAppleEnrollmentProfileFunc NewMDMAppleEnrollmentProfileFunc NewMDMAppleEnrollmentProfileFuncInvoked bool @@ -1580,6 +1605,11 @@ func (s *DataStore) HostIDsByName(ctx context.Context, filter fleet.TeamFilter, return s.HostIDsByNameFunc(ctx, filter, hostnames) } +func (s *DataStore) HostIDsByOSID(ctx context.Context, osID uint, offset int, limit int) ([]uint, error) { + s.HostIDsByOSIDFuncInvoked = true + return s.HostIDsByOSIDFunc(ctx, osID, offset, limit) +} + func (s *DataStore) HostIDsByOSVersion(ctx context.Context, osVersion fleet.OSVersion, offset int, limit int) ([]uint, error) { s.HostIDsByOSVersionFuncInvoked = true return s.HostIDsByOSVersionFunc(ctx, osVersion, offset, limit) @@ -1945,9 +1975,9 @@ func (s *DataStore) ListSoftwareCPEs(ctx context.Context) ([]fleet.SoftwareCPE, return s.ListSoftwareCPEsFunc(ctx) } -func (s *DataStore) InsertVulnerabilities(ctx context.Context, vulns []fleet.SoftwareVulnerability, source fleet.VulnerabilitySource) (int64, error) { - s.InsertVulnerabilitiesFuncInvoked = true - return s.InsertVulnerabilitiesFunc(ctx, vulns, source) +func (s *DataStore) InsertSoftwareVulnerabilities(ctx context.Context, vulns []fleet.SoftwareVulnerability, source fleet.VulnerabilitySource) (int64, error) { + s.InsertSoftwareVulnerabilitiesFuncInvoked = true + return s.InsertSoftwareVulnerabilitiesFunc(ctx, vulns, source) } func (s *DataStore) SoftwareByID(ctx context.Context, id uint, includeCVEScores bool) (*fleet.Software, error) { @@ -2325,11 +2355,31 @@ func (s *DataStore) ProcessList(ctx context.Context) ([]fleet.MySQLProcess, erro return s.ProcessListFunc(ctx) } +func (s *DataStore) ListWindowsUpdatesByHostID(ctx context.Context, hostID uint) ([]fleet.WindowsUpdate, error) { + s.ListWindowsUpdatesByHostIDFuncInvoked = true + return s.ListWindowsUpdatesByHostIDFunc(ctx, hostID) +} + func (s *DataStore) InsertWindowsUpdates(ctx context.Context, hostID uint, updates []fleet.WindowsUpdate) error { s.InsertWindowsUpdatesFuncInvoked = true return s.InsertWindowsUpdatesFunc(ctx, hostID, updates) } +func (s *DataStore) ListOSVulnerabilities(ctx context.Context, hostID []uint) ([]fleet.OSVulnerability, error) { + s.ListOSVulnerabilitiesFuncInvoked = true + return s.ListOSVulnerabilitiesFunc(ctx, hostID) +} + +func (s *DataStore) InsertOSVulnerabilities(ctx context.Context, vulnerabilities []fleet.OSVulnerability, source fleet.VulnerabilitySource) (int64, error) { + s.InsertOSVulnerabilitiesFuncInvoked = true + return s.InsertOSVulnerabilitiesFunc(ctx, vulnerabilities, source) +} + +func (s *DataStore) DeleteOSVulnerabilities(ctx context.Context, vulnerabilities []fleet.OSVulnerability) error { + s.DeleteOSVulnerabilitiesFuncInvoked = true + return s.DeleteOSVulnerabilitiesFunc(ctx, vulnerabilities) +} + func (s *DataStore) NewMDMAppleEnrollmentProfile(ctx context.Context, enrollmentPayload fleet.MDMAppleEnrollmentProfilePayload) (*fleet.MDMAppleEnrollmentProfile, error) { s.NewMDMAppleEnrollmentProfileFuncInvoked = true return s.NewMDMAppleEnrollmentProfileFunc(ctx, enrollmentPayload) diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index 8f0c819ea2..db205bf49e 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -550,7 +550,7 @@ func (s *integrationTestSuite) TestVulnerableSoftware() { soft1 = host.Software[1] } - n, err := s.ds.InsertVulnerabilities( + n, err := s.ds.InsertSoftwareVulnerabilities( context.Background(), []fleet.SoftwareVulnerability{ { SoftwareID: soft1.ID, @@ -4701,7 +4701,7 @@ func (s *integrationTestSuite) TestPaginateListSoftware() { } // add CVEs for the first 10 software, which are the least used (lower hosts_count) - n, err := s.ds.InsertVulnerabilities(context.Background(), vulns, fleet.NVDSource) + n, err := s.ds.InsertSoftwareVulnerabilities(context.Background(), vulns, fleet.NVDSource) require.NoError(t, err) require.Equal(t, 10, int(n)) diff --git a/server/vulnerabilities/msrc/analyzer.go b/server/vulnerabilities/msrc/analyzer.go new file mode 100644 index 0000000000..e578e10f51 --- /dev/null +++ b/server/vulnerabilities/msrc/analyzer.go @@ -0,0 +1,187 @@ +package msrc + +import ( + "context" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + io "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/io" + msrc "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/parsed" + utils "github.com/fleetdm/fleet/v4/server/vulnerabilities/utils" +) + +const ( + hostsBatchSize = 500 + vulnBatchSize = 500 +) + +func Analyze( + ctx context.Context, + ds fleet.Datastore, + os fleet.OperatingSystem, + vulnPath string, + collectVulns bool, +) ([]fleet.OSVulnerability, error) { + bulletin, err := loadBulletin(os, vulnPath) + if err != nil { + return nil, err + } + + // Find matching products inside the bulletin + osProduct := msrc.NewProductFromOS(os) + matchingPIDs := make(map[string]bool) + for pID, p := range bulletin.Products { + if p.Matches(osProduct) { + matchingPIDs[pID] = true + } + } + + if len(matchingPIDs) == 0 { + return nil, nil + } + + toInsert := make(map[string]fleet.OSVulnerability) + toDelete := make(map[string]fleet.OSVulnerability) + + var offset int + for { + hIDs, err := ds.HostIDsByOSID(ctx, os.ID, offset, hostsBatchSize) + if err != nil { + return nil, err + } + + if len(hIDs) == 0 { + break + } + + offset += len(hIDs) + + // Run vulnerability detection for all hosts in this batch (hIDs) + // and store the results in 'found'. + found := make(map[uint][]fleet.OSVulnerability, len(hIDs)) + for _, hID := range hIDs { + updates, err := ds.ListWindowsUpdatesByHostID(ctx, hID) + if err != nil { + return nil, err + } + + var vs []fleet.OSVulnerability + for cve, v := range bulletin.Vulnerabities { + // Check if this vulnerability targets the OS + if !utils.ProductIDsIntersect(v.ProductIDs, matchingPIDs) { + continue + } + if patched(os, bulletin, v, matchingPIDs, updates) { + continue + } + vs = append(vs, fleet.OSVulnerability{OSID: os.ID, HostID: hID, CVE: cve}) + } + found[hID] = vs + } + + // Fetch all stored vulnerabilities for the current batch + osVulns, err := ds.ListOSVulnerabilities(ctx, hIDs) + if err != nil { + return nil, err + } + existing := make(map[uint][]fleet.OSVulnerability) + for _, osv := range osVulns { + existing[osv.HostID] = append(existing[osv.HostID], osv) + } + + // Compute what needs to be inserted/deleted for this batch + for _, hID := range hIDs { + insrt, del := utils.VulnsDelta(found[hID], existing[hID]) + for _, i := range insrt { + toInsert[i.Key()] = i + } + for _, d := range del { + toDelete[d.Key()] = d + } + } + } + + err = utils.BatchProcess(toDelete, func(v []fleet.OSVulnerability) error { + return ds.DeleteOSVulnerabilities(ctx, v) + }, vulnBatchSize) + if err != nil { + return nil, err + } + + var inserted []fleet.OSVulnerability + if collectVulns { + inserted = make([]fleet.OSVulnerability, 0, len(toInsert)) + } + + err = utils.BatchProcess(toInsert, func(v []fleet.OSVulnerability) error { + n, err := ds.InsertOSVulnerabilities(ctx, v, fleet.MSRCSource) + if err != nil { + return err + } + + if collectVulns && n > 0 { + inserted = append(inserted, v...) + } + + return nil + }, vulnBatchSize) + if err != nil { + return nil, err + } + + return inserted, nil +} + +// patched returns true if the vulnerability (v) is patched by the any of the provided Windows +// updates. +func patched( + os fleet.OperatingSystem, + b *msrc.SecurityBulletin, + v msrc.Vulnerability, + matchingPIDs map[string]bool, + updates []fleet.WindowsUpdate, +) bool { + // check if any update directly remediates the vulnerability, + // this will be much faster than walking the forest of vendor fixes. + for _, u := range updates { + if v.RemediatedBy[u.KBID] { + return true + } + } + + for KBID := range v.RemediatedBy { + fix := b.VendorFixes[KBID] + + // Check if this vendor fix targets the OS + if !utils.ProductIDsIntersect(fix.ProductIDs, matchingPIDs) { + continue + } + + // Check if the kernel build already contains the fix + if utils.Rpmvercmp(os.KernelVersion, fix.FixedBuild) >= 0 { + return true + } + + // If not, walk the forest + for _, u := range updates { + if b.KBIDsConnected(KBID, u.KBID) { + return true + } + } + } + + return false +} + +// loadBulletin loads the most recent bulletin for the given os +func loadBulletin(os fleet.OperatingSystem, dir string) (*msrc.SecurityBulletin, error) { + product := msrc.NewProductFromOS(os) + fileName := io.FileName(product.Name(), time.Now()) + + latest, err := utils.LatestFile(fileName, dir) + if err != nil { + return nil, err + } + + return msrc.UnmarshalBulletin(latest) +} diff --git a/server/vulnerabilities/msrc/analyzer_test.go b/server/vulnerabilities/msrc/analyzer_test.go new file mode 100644 index 0000000000..7f740d3c1f --- /dev/null +++ b/server/vulnerabilities/msrc/analyzer_test.go @@ -0,0 +1,130 @@ +package msrc + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/ptr" + io "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/io" + "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/parsed" + "github.com/stretchr/testify/require" +) + +func TestAnalyzer(t *testing.T) { + op := fleet.OperatingSystem{ + Name: "Microsoft Windows 11 Enterprise Evaluation", + Version: "21H2", + Arch: "64-bit", + KernelVersion: "10.0.22000.795", + Platform: "windows", + } + prod := parsed.NewProductFromOS(op) + + t.Run("#patched", func(t *testing.T) { + t.Run("no updates", func(t *testing.T) { + b := parsed.NewSecurityBulletin(prod.Name()) + b.Products["123"] = prod + b.Vulnerabities["cve-123"] = parsed.NewVulnerability(nil) + pIDs := map[string]bool{"123": true} + require.False(t, patched(op, b, b.Vulnerabities["cve-123"], pIDs, nil)) + }) + + t.Run("directly remediated", func(t *testing.T) { + b := parsed.NewSecurityBulletin(prod.Name()) + b.Products["123"] = prod + + vuln := parsed.NewVulnerability(nil) + vuln.RemediatedBy[123] = true + b.Vulnerabities["cve-123"] = vuln + + pIDs := map[string]bool{"123": true} + + updates := []fleet.WindowsUpdate{ + {KBID: 123}, + {KBID: 456}, + } + + require.True(t, patched(op, b, b.Vulnerabities["cve-123"], pIDs, updates)) + }) + + t.Run("remediated by build", func(t *testing.T) { + b := parsed.NewSecurityBulletin(prod.Name()) + b.Products["123"] = prod + pIDs := map[string]bool{"123": true} + + vuln := parsed.NewVulnerability(nil) + vuln.RemediatedBy[456] = true + b.Vulnerabities["cve-123"] = vuln + + vfA := parsed.NewVendorFix("10.0.22000.794") + vfA.Supersedes = ptr.Uint(123) + vfA.ProductIDs["123"] = true + b.VendorFixes[456] = vfA + + updates := []fleet.WindowsUpdate{ + {KBID: 789}, + } + + require.True(t, patched(op, b, b.Vulnerabities["cve-123"], pIDs, updates)) + }) + + t.Run("remediated by a cumulative update", func(t *testing.T) { + b := parsed.NewSecurityBulletin(prod.Name()) + b.Products["123"] = prod + pIDs := map[string]bool{"123": true} + + vuln := parsed.NewVulnerability(nil) + vuln.RemediatedBy[456] = true + b.Vulnerabities["cve-123"] = vuln + + vfA := parsed.NewVendorFix("10.0.22000.796") + vfA.Supersedes = ptr.Uint(123) + vfA.ProductIDs["123"] = true + b.VendorFixes[456] = vfA + + vfB := parsed.NewVendorFix("10.0.22000.796") + vfB.Supersedes = ptr.Uint(456) + vfB.ProductIDs["123"] = true + b.VendorFixes[789] = vfA + + updates := []fleet.WindowsUpdate{ + {KBID: 789}, + } + + require.True(t, patched(op, b, b.Vulnerabities["cve-123"], pIDs, updates)) + }) + }) + + t.Run("#loadBulletin", func(t *testing.T) { + t.Run("dir does not exists", func(t *testing.T) { + bulletin, err := loadBulletin(op, "over_the_rainbow") + require.Error(t, err) + require.Nil(t, bulletin) + }) + + t.Run("returns the lastest bulletin", func(t *testing.T) { + d := time.Now() + dir := t.TempDir() + + b := parsed.NewSecurityBulletin(prod.Name()) + b.Products["1235"] = prod + + fileName := io.FileName(b.ProductName, d) + filePath := filepath.Join(dir, fileName) + + payload, err := json.Marshal(b) + require.NoError(t, err) + + err = os.WriteFile(filePath, payload, 0o644) + require.NoError(t, err) + + actual, err := loadBulletin(op, dir) + require.NoError(t, err) + require.Equal(t, prod.Name(), actual.ProductName) + }) + }) +} diff --git a/server/vulnerabilities/msrc/io/github.go b/server/vulnerabilities/msrc/io/github.go index 1e6611e7ce..ad191bfd55 100644 --- a/server/vulnerabilities/msrc/io/github.go +++ b/server/vulnerabilities/msrc/io/github.go @@ -27,7 +27,7 @@ type ReleaseLister interface { // GitHubAPI allows users to interact with the MSRC artifacts published on Github. type GitHubAPI interface { Download(string) (string, error) - Bulletins() (map[SecurityBulletinName]string, error) + Bulletins(context.Context) (map[SecurityBulletinName]string, error) } type GitHubClient struct { @@ -63,8 +63,8 @@ func (gh GitHubClient) Download(URL string) (string, error) { } // Bulletins returns a map of 'bulletin name' => 'download URL' of the bulletins stored as assets on Github. -func (gh GitHubClient) Bulletins() (map[SecurityBulletinName]string, error) { - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) +func (gh GitHubClient) Bulletins(ctx context.Context) (map[SecurityBulletinName]string, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() releases, r, err := gh.releases.ListReleases( diff --git a/server/vulnerabilities/msrc/io/github_test.go b/server/vulnerabilities/msrc/io/github_test.go index 79aa53c160..aa730d0afc 100644 --- a/server/vulnerabilities/msrc/io/github_test.go +++ b/server/vulnerabilities/msrc/io/github_test.go @@ -72,6 +72,8 @@ func (m mockGHReleaseLister) ListReleases( } func TestGithubClient(t *testing.T) { + ctx := context.Background() + t.Run("#Download", func(t *testing.T) { fileName := fmt.Sprintf("%sWindows_11-2022_09_10.json", mSRCFilePrefix) urlPath := fmt.Sprintf("/fleetdm/nvd/releases/download/202208290017/%s", fileName) @@ -99,7 +101,7 @@ func TestGithubClient(t *testing.T) { t.Run("#Bulletins", func(t *testing.T) { sut := NewGitHubClient(nil, mockGHReleaseLister{}, t.TempDir()) - bulletins, err := sut.Bulletins() + bulletins, err := sut.Bulletins(ctx) require.NoError(t, err) require.Len(t, bulletins, 2) diff --git a/server/vulnerabilities/msrc/io/security_bulletin_name.go b/server/vulnerabilities/msrc/io/security_bulletin_name.go index fc61262241..9898aac40d 100644 --- a/server/vulnerabilities/msrc/io/security_bulletin_name.go +++ b/server/vulnerabilities/msrc/io/security_bulletin_name.go @@ -32,9 +32,9 @@ func (sbn SecurityBulletinName) date() (time.Time, error) { return time.Parse(dateLayout, timeRaw) } -func FileName(productName string, date time.Time, ext string) string { +func FileName(productName string, date time.Time) string { pName := strings.Replace(productName, " ", "_", -1) - return fmt.Sprintf("%s%s-%d_%02d_%02d.%s", mSRCFilePrefix, pName, date.Year(), date.Month(), date.Day(), ext) + return fmt.Sprintf("%s%s-%d_%02d_%02d.%s", mSRCFilePrefix, pName, date.Year(), date.Month(), date.Day(), fileExt) } func (sbn SecurityBulletinName) Before(other SecurityBulletinName) bool { diff --git a/server/vulnerabilities/msrc/parsed/product.go b/server/vulnerabilities/msrc/parsed/product.go index d9eaefda52..1d81746067 100644 --- a/server/vulnerabilities/msrc/parsed/product.go +++ b/server/vulnerabilities/msrc/parsed/product.go @@ -1,16 +1,25 @@ package parsed -import "strings" +import ( + "fmt" + "strings" + + "github.com/fleetdm/fleet/v4/server/fleet" +) // Product abstracts a MS full product name. // A full product name includes the name of the product plus its arch // (if any) and its version (if any). type Product string -func NewProduct(fullName string) Product { +func NewProductFromFullName(fullName string) Product { return Product(fullName) } +func NewProductFromOS(os fleet.OperatingSystem) Product { + return Product(fmt.Sprintf("%s %s for %s", os.Name, os.Version, os.Arch)) +} + // Arch returns the archicture for the current Microsoft product, if none can // be found then "all" is returned. Returned values are meant to match the values returned from // `SELECT arch FROM os_version` in OSQuery. @@ -19,10 +28,13 @@ func NewProduct(fullName string) Product { func (p Product) Arch() string { val := string(p) switch { - case strings.Index(val, "32-bit") != -1: - return "32-bit" - case strings.Index(val, "x64") != -1: + case strings.Index(val, "x64") != -1 || + strings.Index(val, "64-bit") != -1 || + strings.Index(val, "x86_64") != -1: return "64-bit" + case strings.Index(val, "32-bit") != -1 || + strings.Index(val, "x86") != -1: + return "32-bit" case strings.Index(val, "ARM64") != -1: return "arm64" case strings.Index(val, "Itanium-Based") != -1: @@ -75,3 +87,13 @@ func (p Product) Name() string { return "" } } + +// Matches checks whehter product A matches product B by checking to see if both are for the same +// product and if the architecture they target are compatible. This function is commutative. +func (p Product) Matches(o Product) bool { + if p.Name() != o.Name() { + return false + } + + return p.Arch() == "all" || o.Arch() == "all" || p.Arch() == o.Arch() +} diff --git a/server/vulnerabilities/msrc/parsed/product_test.go b/server/vulnerabilities/msrc/parsed/product_test.go index 854c512904..d64681e60c 100644 --- a/server/vulnerabilities/msrc/parsed/product_test.go +++ b/server/vulnerabilities/msrc/parsed/product_test.go @@ -3,9 +3,67 @@ package parsed import ( "testing" + "github.com/fleetdm/fleet/v4/server/fleet" "github.com/stretchr/testify/require" ) +func TestNewProductFromOS(t *testing.T) { + os := fleet.OperatingSystem{ + Name: "Microsoft Windows 11 Enterprise Evaluation", + Version: "21H2", + Arch: "64-bit", + KernelVersion: "10.0.22000.795", + Platform: "windows", + } + + pA := NewProductFromOS(os) + pB := NewProductFromFullName("Windows 11 for x64-based Systems") + + require.Equal(t, "Windows 11", pA.Name()) + require.Equal(t, "64-bit", pA.Arch()) + + require.True(t, pA.Matches(pB)) +} + +func TestMatches(t *testing.T) { + t.Run("from differect products", func(t *testing.T) { + pA := NewProductFromFullName("Windows 10 Version 1809 for ARM64-based Systems") + pB := NewProductFromFullName("Windows 11 for x64-based Systems") + + require.False(t, pA.Matches(pB)) + require.False(t, pB.Matches(pA)) + }) + + t.Run("from differect arch", func(t *testing.T) { + pA := NewProductFromFullName("Windows 11 for ARM64-based Systems") + pB := NewProductFromFullName("Windows 11 for x64-based Systems") + + require.False(t, pA.Matches(pB)) + require.False(t, pB.Matches(pA)) + }) + + t.Run("same product but for different architecture", func(t *testing.T) { + pA := NewProductFromFullName("Windows 10 Version 1809 for ARM64-based Systems") + pB := NewProductFromFullName("Windows 10 Version 1809 for x64-based Systems") + require.False(t, pA.Matches(pB)) + require.False(t, pB.Matches(pA)) + }) + + t.Run("same product one with no architecture", func(t *testing.T) { + pA := NewProductFromFullName("Windows 10 Version 1809") + pB := NewProductFromFullName("Windows 10 Version 1809 for x64-based Systems") + require.True(t, pA.Matches(pB)) + require.True(t, pB.Matches(pA)) + }) + + t.Run("same product same arch", func(t *testing.T) { + pA := NewProductFromFullName("Windows 10 Version 1809 for x64-based Systems") + pB := NewProductFromFullName("Windows 10 Version 1809 for x64-based Systems") + require.True(t, pA.Matches(pB)) + require.True(t, pB.Matches(pA)) + }) +} + func TestFullProductName(t *testing.T) { testCases := []struct { fullName string @@ -361,14 +419,14 @@ func TestFullProductName(t *testing.T) { t.Run("#ArchFromProdName", func(t *testing.T) { for _, tCase := range testCases { - sut := NewProduct(tCase.fullName) + sut := NewProductFromFullName(tCase.fullName) require.Equal(t, tCase.arch, sut.Arch(), tCase) } }) t.Run("#NameFromFullProdName", func(t *testing.T) { for _, tCase := range testCases { - sut := NewProduct(tCase.fullName) + sut := NewProductFromFullName(tCase.fullName) require.Equal(t, tCase.prodName, sut.Name(), tCase) } }) diff --git a/server/vulnerabilities/msrc/parsed/security_bulletin.go b/server/vulnerabilities/msrc/parsed/security_bulletin.go index c02f0c689e..d5c425ac86 100644 --- a/server/vulnerabilities/msrc/parsed/security_bulletin.go +++ b/server/vulnerabilities/msrc/parsed/security_bulletin.go @@ -15,19 +15,22 @@ type SecurityBulletin struct { // We can have many different 'products' under a single name, for example, for 'Windows 10': // - Windows 10 Version 1809 for 32-bit Systems // - Windows 10 Version 1909 for x64-based Systems - Products map[string]string + Products map[string]Product // All vulnerabilities contained in this bulletin, by CVE Vulnerabities map[string]Vulnerability // All vendor fixes for remediating the vulnerabilities contained in this bulletin, by KBID - VendorFixes map[int]VendorFix + VendorFixes map[uint]VendorFix + + // Data struct used for telling if two KBID are 'connected' + vfForest *weightedUnionFind } func NewSecurityBulletin(pName string) *SecurityBulletin { return &SecurityBulletin{ ProductName: pName, - Products: make(map[string]string), + Products: make(map[string]Product), Vulnerabities: make(map[string]Vulnerability), - VendorFixes: make(map[int]VendorFix), + VendorFixes: make(map[uint]VendorFix), } } @@ -80,7 +83,7 @@ func (b *SecurityBulletin) Merge(other *SecurityBulletin) error { newVF.ProductIDs[pID] = v } if r.Supersedes != nil { - newVF.Supersedes = ptr.Int(*r.Supersedes) + newVF.Supersedes = ptr.Uint(*r.Supersedes) } b.VendorFixes[kbID] = newVF } @@ -89,28 +92,121 @@ func (b *SecurityBulletin) Merge(other *SecurityBulletin) error { return nil } +func (b *SecurityBulletin) initUnionFind() *weightedUnionFind { + uf := &weightedUnionFind{} + + uf.ids = make(map[uint]uint, len(b.VendorFixes)) + uf.size = make(map[uint]uint16, len(b.VendorFixes)) + + // Init forest + for KBID := range b.VendorFixes { + uf.ids[KBID] = KBID + uf.size[KBID] = 1 + } + + // Create unions + for KBID, vf := range b.VendorFixes { + if vf.Supersedes != nil { + uf.union(KBID, *vf.Supersedes) + } + } + + return uf +} + +func (b *SecurityBulletin) getVFForest() *weightedUnionFind { + if b.vfForest == nil { + b.vfForest = b.initUnionFind() + } + return b.vfForest +} + +// KBIDsConnected returns whether two updates are 'connected', used for dealing with cumulative +// updates. A cumulative update can replace another update (we determine this via the 'Supersedes' +// prop. in the VendorFix type), when determining whether a host is susceptible to a vulnerability we +// are interested in determining whether the host has a specific update installed or any of the +// superseded updates. +func (b *SecurityBulletin) KBIDsConnected(p, q uint) bool { + return b.getVFForest().connected(p, q) +} + +// ---- +// UnionFind +// ---- + +// We will be using a weighted union-find data struct for determining whether two KBIDs are 'connected', +// this will be used for handling cumulative updates. +type weightedUnionFind struct { + // Each 'value' points to the parent of 'key', each key is a KBID + ids map[uint]uint + // The size of each tree by 'KBID' + size map[uint]uint16 +} + +// union connects two components (KBID) +func (uf *weightedUnionFind) union(p uint, q uint) { + pRoot := uf.root(p) + qRoot := uf.root(q) + + if uf.size[qRoot] < uf.size[pRoot] { + uf.ids[qRoot] = uf.ids[pRoot] + uf.size[pRoot] += uf.size[qRoot] + } else { + uf.ids[pRoot] = uf.ids[qRoot] + uf.size[qRoot] += uf.size[pRoot] + } +} + +// root returns the root of the 'p' tree +func (uf *weightedUnionFind) root(p uint) uint { + if _, ok := uf.ids[p]; !ok { + return p + } + + for uf.ids[p] != p { + uf.ids[p] = uf.ids[uf.ids[p]] + p = uf.ids[p] + } + + return p +} + +// connected returns whether two components are connected, for example: +// A -> B -> C -> D; connected(A, C) -> true +func (uf *weightedUnionFind) connected(p uint, q uint) bool { + return uf.root(p) == uf.root(q) +} + +// ---------------------- +// Vulnerability +// ---------------------- + type Vulnerability struct { PublishedEpoch *int64 - // Set of products that are susceptible to this vuln. + // Set of products ids that are susceptible to this vuln. ProductIDs map[string]bool // Set of Vendor fixes that remediate this vuln. - RemediatedBy map[int]bool + RemediatedBy map[uint]bool } func NewVulnerability(publishedDateEpoch *int64) Vulnerability { return Vulnerability{ PublishedEpoch: publishedDateEpoch, ProductIDs: make(map[string]bool), - RemediatedBy: make(map[int]bool), + RemediatedBy: make(map[uint]bool), } } +// ---------------------- +// VendorFix +// ---------------------- + type VendorFix struct { - // TODO (juan): Do we need this? FixedBuild string + // Set of products ids that target this vendor fix ProductIDs map[string]bool // A Reference to what vendor fix this particular vendor fix 'replaces'. - Supersedes *int `json:",omitempty"` + Supersedes *uint `json:",omitempty"` } func NewVendorFix(fixedBuild string) VendorFix { diff --git a/server/vulnerabilities/msrc/parsed/security_bulletin_test.go b/server/vulnerabilities/msrc/parsed/security_bulletin_test.go index a041df751a..4ae9b43338 100644 --- a/server/vulnerabilities/msrc/parsed/security_bulletin_test.go +++ b/server/vulnerabilities/msrc/parsed/security_bulletin_test.go @@ -32,28 +32,28 @@ func TestSecurityBulletin(t *testing.T) { a.Merge(b) - require.Equal(t, a.Products["123"], "Windows 10 A") - require.Equal(t, a.Products["456"], "Windows 10 B") - require.Equal(t, a.Products["780"], "Windows 10 C") - require.Equal(t, a.Products["980"], "Windows 10 D") + require.Equal(t, a.Products["123"], NewProductFromFullName("Windows 10 A")) + require.Equal(t, a.Products["456"], NewProductFromFullName("Windows 10 B")) + require.Equal(t, a.Products["780"], NewProductFromFullName("Windows 10 C")) + require.Equal(t, a.Products["980"], NewProductFromFullName("Windows 10 D")) }) t.Run(".Vulnerabities", func(t *testing.T) { cve1 := NewVulnerability(ptr.Int64(123)) cve1.ProductIDs = map[string]bool{"111": true, "222": true} - cve1.RemediatedBy = map[int]bool{1: true} + cve1.RemediatedBy = map[uint]bool{1: true} cve2 := NewVulnerability(ptr.Int64(456)) cve2.ProductIDs = map[string]bool{"333": true, "444": true} - cve2.RemediatedBy = map[int]bool{2: true} + cve2.RemediatedBy = map[uint]bool{2: true} cve3 := NewVulnerability(ptr.Int64(555)) cve3.ProductIDs = map[string]bool{"aaa": true, "bbb": true} - cve3.RemediatedBy = map[int]bool{3: true} + cve3.RemediatedBy = map[uint]bool{3: true} cve4 := NewVulnerability(ptr.Int64(777)) cve4.ProductIDs = map[string]bool{"ccc": true, "ddd": true} - cve3.RemediatedBy = map[int]bool{4: true} + cve3.RemediatedBy = map[uint]bool{4: true} a := NewSecurityBulletin("Windows 10") a.Vulnerabities["cve-1"] = cve1 @@ -82,13 +82,13 @@ func TestSecurityBulletin(t *testing.T) { }) t.Run(".VendorFixes", func(t *testing.T) { - vf1 := NewVendorFix("1") + vf1 := NewVendorFix("") vf1.ProductIDs = map[string]bool{"111": true, "222": true} - vf1.Supersedes = ptr.Int(1) + vf1.Supersedes = ptr.Uint(1) - vf2 := NewVendorFix("2") + vf2 := NewVendorFix("") vf2.ProductIDs = map[string]bool{"333": true, "444": true} - vf2.Supersedes = ptr.Int(2) + vf2.Supersedes = ptr.Uint(2) a := NewSecurityBulletin("Windows 10") a.VendorFixes[1] = vf1 @@ -98,8 +98,8 @@ func TestSecurityBulletin(t *testing.T) { a.Merge(b) - require.Equal(t, *a.VendorFixes[1].Supersedes, int(1)) - require.Equal(t, *a.VendorFixes[2].Supersedes, int(2)) + require.Equal(t, *a.VendorFixes[1].Supersedes, uint(1)) + require.Equal(t, *a.VendorFixes[2].Supersedes, uint(2)) require.Equal(t, a.VendorFixes[1].ProductIDs, vf1.ProductIDs) require.Equal(t, a.VendorFixes[2].ProductIDs, vf2.ProductIDs) diff --git a/server/vulnerabilities/msrc/parser.go b/server/vulnerabilities/msrc/parser.go index 6a208b6d6c..ca020e8954 100644 --- a/server/vulnerabilities/msrc/parser.go +++ b/server/vulnerabilities/msrc/parser.go @@ -7,6 +7,7 @@ import ( "os" "strconv" + "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/parsed" msrcxml "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/xml" ) @@ -37,7 +38,7 @@ func mapToSecurityBulletins(rXML *msrcxml.FeedResult) (map[string]*parsed.Securi pIDToPName := make(map[string]string, len(rXML.WinProducts)) for pID, p := range rXML.WinProducts { - name := parsed.NewProduct(p.FullName).Name() + name := parsed.NewProductFromFullName(p.FullName).Name() // If the name could not be determined means that we have an un-supported Windows product if name == "" { continue @@ -46,7 +47,7 @@ func mapToSecurityBulletins(rXML *msrcxml.FeedResult) (map[string]*parsed.Securi if bulletins[name] == nil { bulletins[name] = parsed.NewSecurityBulletin(name) } - bulletins[name].Products[pID] = p.FullName + bulletins[name].Products[pID] = parsed.NewProductFromFullName(p.FullName) pIDToPName[pID] = name } @@ -59,19 +60,26 @@ func mapToSecurityBulletins(rXML *msrcxml.FeedResult) (map[string]*parsed.Securi // We assume that rem.Description will contain the ID portion of a KBID, which should // be always a numeric value. - remediatedKBID, err := strconv.Atoi(rem.Description) + remediatedKBIDRaw, err := strconv.Atoi(rem.Description) if err != nil { return nil, fmt.Errorf("invalid remediation KBID %q for %s", rem.Description, v.CVE) } + if remediatedKBIDRaw < 0 { + return nil, fmt.Errorf("invalid remediation KBID %q for %s", rem.Description, v.CVE) + } + remediatedKBID := uint(remediatedKBIDRaw) // rem.Supercedence should have the ID portion of a KBID which the current vendor fix replaces. - var supersedes *int + var supersedes *uint if rem.Supercedence != "" { r, err := strconv.Atoi(rem.Supercedence) if err != nil { return nil, fmt.Errorf("invalid supercedence KBID %q for %s", rem.Supercedence, v.CVE) } - supersedes = &r + if r < 0 { + return nil, fmt.Errorf("invalid supercedence KBID %q for %s", rem.Supercedence, v.CVE) + } + supersedes = ptr.Uint(uint(r)) } for _, pID := range rem.ProductIDs { @@ -88,6 +96,8 @@ func mapToSecurityBulletins(rXML *msrcxml.FeedResult) (map[string]*parsed.Securi if vuln, ok = b.Vulnerabities[v.CVE]; !ok { vuln = parsed.NewVulnerability(v.PublishedDateEpoch()) } + // At this point we know that the remediation is a vendor fix that targets a windows + // product, so we add the remediation's product ID to the vulnerability's targeted products. vuln.ProductIDs[pID] = true vuln.RemediatedBy[remediatedKBID] = true diff --git a/server/vulnerabilities/msrc/parser_test.go b/server/vulnerabilities/msrc/parser_test.go index 94fc6ce68d..b003af2f0a 100644 --- a/server/vulnerabilities/msrc/parser_test.go +++ b/server/vulnerabilities/msrc/parser_test.go @@ -40,75 +40,75 @@ func TestParser(t *testing.T) { require.NoError(t, err) // All the products we expect to see, grouped by their product name - expectedProducts := map[string]map[string]string{ + expectedProducts := map[string]map[string]parsed.Product{ "Windows 10": { - "11568": "Windows 10 Version 1809 for 32-bit Systems", - "11569": "Windows 10 Version 1809 for x64-based Systems", - "11570": "Windows 10 Version 1809 for ARM64-based Systems", - "11712": "Windows 10 Version 1909 for 32-bit Systems", - "11713": "Windows 10 Version 1909 for x64-based Systems", - "11714": "Windows 10 Version 1909 for ARM64-based Systems", - "11896": "Windows 10 Version 21H1 for x64-based Systems", - "11897": "Windows 10 Version 21H1 for ARM64-based Systems", - "11898": "Windows 10 Version 21H1 for 32-bit Systems", - "11800": "Windows 10 Version 20H2 for x64-based Systems", - "11801": "Windows 10 Version 20H2 for 32-bit Systems", - "11802": "Windows 10 Version 20H2 for ARM64-based Systems", - "11929": "Windows 10 Version 21H2 for 32-bit Systems", - "11930": "Windows 10 Version 21H2 for ARM64-based Systems", - "11931": "Windows 10 Version 21H2 for x64-based Systems", - "10729": "Windows 10 for 32-bit Systems", - "10735": "Windows 10 for x64-based Systems", - "10852": "Windows 10 Version 1607 for 32-bit Systems", - "10853": "Windows 10 Version 1607 for x64-based Systems", + "11568": parsed.NewProductFromFullName("Windows 10 Version 1809 for 32-bit Systems"), + "11569": parsed.NewProductFromFullName("Windows 10 Version 1809 for x64-based Systems"), + "11570": parsed.NewProductFromFullName("Windows 10 Version 1809 for ARM64-based Systems"), + "11712": parsed.NewProductFromFullName("Windows 10 Version 1909 for 32-bit Systems"), + "11713": parsed.NewProductFromFullName("Windows 10 Version 1909 for x64-based Systems"), + "11714": parsed.NewProductFromFullName("Windows 10 Version 1909 for ARM64-based Systems"), + "11896": parsed.NewProductFromFullName("Windows 10 Version 21H1 for x64-based Systems"), + "11897": parsed.NewProductFromFullName("Windows 10 Version 21H1 for ARM64-based Systems"), + "11898": parsed.NewProductFromFullName("Windows 10 Version 21H1 for 32-bit Systems"), + "11800": parsed.NewProductFromFullName("Windows 10 Version 20H2 for x64-based Systems"), + "11801": parsed.NewProductFromFullName("Windows 10 Version 20H2 for 32-bit Systems"), + "11802": parsed.NewProductFromFullName("Windows 10 Version 20H2 for ARM64-based Systems"), + "11929": parsed.NewProductFromFullName("Windows 10 Version 21H2 for 32-bit Systems"), + "11930": parsed.NewProductFromFullName("Windows 10 Version 21H2 for ARM64-based Systems"), + "11931": parsed.NewProductFromFullName("Windows 10 Version 21H2 for x64-based Systems"), + "10729": parsed.NewProductFromFullName("Windows 10 for 32-bit Systems"), + "10735": parsed.NewProductFromFullName("Windows 10 for x64-based Systems"), + "10852": parsed.NewProductFromFullName("Windows 10 Version 1607 for 32-bit Systems"), + "10853": parsed.NewProductFromFullName("Windows 10 Version 1607 for x64-based Systems"), }, "Windows Server 2019": { - "11571": "Windows Server 2019", - "11572": "Windows Server 2019 (Server Core installation)", + "11571": parsed.NewProductFromFullName("Windows Server 2019"), + "11572": parsed.NewProductFromFullName("Windows Server 2019 (Server Core installation)"), }, "Windows Server 2022": { - "11923": "Windows Server 2022", - "11924": "Windows Server 2022 (Server Core installation)", + "11923": parsed.NewProductFromFullName("Windows Server 2022"), + "11924": parsed.NewProductFromFullName("Windows Server 2022 (Server Core installation)"), }, "Windows Server": { - "11803": "Windows Server, version 20H2 (Server Core Installation)", + "11803": parsed.NewProductFromFullName("Windows Server, version 20H2 (Server Core Installation)"), }, "Windows 11": { - "11926": "Windows 11 for x64-based Systems", - "11927": "Windows 11 for ARM64-based Systems", + "11926": parsed.NewProductFromFullName("Windows 11 for x64-based Systems"), + "11927": parsed.NewProductFromFullName("Windows 11 for ARM64-based Systems"), }, "Windows Server 2016": { - "10816": "Windows Server 2016", - "10855": "Windows Server 2016 (Server Core installation)", + "10816": parsed.NewProductFromFullName("Windows Server 2016"), + "10855": parsed.NewProductFromFullName("Windows Server 2016 (Server Core installation)"), }, "Windows 8.1": { - "10481": "Windows 8.1 for 32-bit systems", - "10482": "Windows 8.1 for x64-based systems", + "10481": parsed.NewProductFromFullName("Windows 8.1 for 32-bit systems"), + "10482": parsed.NewProductFromFullName("Windows 8.1 for x64-based systems"), }, "Windows RT 8.1": { - "10484": "Windows RT 8.1", + "10484": parsed.NewProductFromFullName("Windows RT 8.1"), }, "Windows Server 2012": { - "10378": "Windows Server 2012", - "10379": "Windows Server 2012 (Server Core installation)", + "10378": parsed.NewProductFromFullName("Windows Server 2012"), + "10379": parsed.NewProductFromFullName("Windows Server 2012 (Server Core installation)"), }, "Windows Server 2012 R2": { - "10483": "Windows Server 2012 R2", - "10543": "Windows Server 2012 R2 (Server Core installation)", + "10483": parsed.NewProductFromFullName("Windows Server 2012 R2"), + "10543": parsed.NewProductFromFullName("Windows Server 2012 R2 (Server Core installation)"), }, "Windows 7": { - "10047": "Windows 7 for 32-bit Systems Service Pack 1", - "10048": "Windows 7 for x64-based Systems Service Pack 1", + "10047": parsed.NewProductFromFullName("Windows 7 for 32-bit Systems Service Pack 1"), + "10048": parsed.NewProductFromFullName("Windows 7 for x64-based Systems Service Pack 1"), }, "Windows Server 2008": { - "9312": "Windows Server 2008 for 32-bit Systems Service Pack 2", - "10287": "Windows Server 2008 for 32-bit Systems Service Pack 2 (Server Core installation)", - "9318": "Windows Server 2008 for x64-based Systems Service Pack 2", - "9344": "Windows Server 2008 for x64-based Systems Service Pack 2 (Server Core installation)", + "9312": parsed.NewProductFromFullName("Windows Server 2008 for 32-bit Systems Service Pack 2"), + "10287": parsed.NewProductFromFullName("Windows Server 2008 for 32-bit Systems Service Pack 2 (Server Core installation)"), + "9318": parsed.NewProductFromFullName("Windows Server 2008 for x64-based Systems Service Pack 2"), + "9344": parsed.NewProductFromFullName("Windows Server 2008 for x64-based Systems Service Pack 2 (Server Core installation)"), }, "Windows Server 2008 R2": { - "10051": "Windows Server 2008 R2 for x64-based Systems Service Pack 1", - "10049": "Windows Server 2008 R2 for x64-based Systems Service Pack 1 (Server Core installation)", + "10051": parsed.NewProductFromFullName("Windows Server 2008 R2 for x64-based Systems Service Pack 1"), + "10049": parsed.NewProductFromFullName("Windows Server 2008 R2 for x64-based Systems Service Pack 1 (Server Core installation)"), }, } @@ -736,7 +736,7 @@ func TestParser(t *testing.T) { "10852": true, "10853": true, }, - RemediatedBy: map[int]bool{ + RemediatedBy: map[uint]bool{ 5013941: true, 5013952: true, 5013942: true, @@ -752,7 +752,7 @@ func TestParser(t *testing.T) { "11571": true, "11572": true, }, - RemediatedBy: map[int]bool{ + RemediatedBy: map[uint]bool{ 5013941: true, }, }, @@ -765,7 +765,7 @@ func TestParser(t *testing.T) { "11923": true, "11924": true, }, - RemediatedBy: map[int]bool{ + RemediatedBy: map[uint]bool{ 5013944: true, }, }, @@ -777,7 +777,7 @@ func TestParser(t *testing.T) { ProductIDs: map[string]bool{ "11803": true, }, - RemediatedBy: map[int]bool{ + RemediatedBy: map[uint]bool{ 5013942: true, }, }, @@ -792,7 +792,7 @@ func TestParser(t *testing.T) { "9318": true, "9344": true, }, - RemediatedBy: map[int]bool{ + RemediatedBy: map[uint]bool{ 5014010: true, 5014006: true, }, @@ -806,7 +806,7 @@ func TestParser(t *testing.T) { "10051": true, "10049": true, }, - RemediatedBy: map[int]bool{ + RemediatedBy: map[uint]bool{ 5014012: true, 5013999: true, }, @@ -820,7 +820,7 @@ func TestParser(t *testing.T) { "10378": true, "10379": true, }, - RemediatedBy: map[int]bool{ + RemediatedBy: map[uint]bool{ 5014017: true, 5014018: true, }, @@ -834,7 +834,7 @@ func TestParser(t *testing.T) { "10483": true, "10543": true, }, - RemediatedBy: map[int]bool{ + RemediatedBy: map[uint]bool{ 5014011: true, 5014001: true, }, @@ -848,7 +848,7 @@ func TestParser(t *testing.T) { "10047": true, "10048": true, }, - RemediatedBy: map[int]bool{ + RemediatedBy: map[uint]bool{ 5014012: true, 5013999: true, }, @@ -862,7 +862,7 @@ func TestParser(t *testing.T) { "10816": true, "10855": true, }, - RemediatedBy: map[int]bool{ + RemediatedBy: map[uint]bool{ 5013952: true, }, }, @@ -875,7 +875,7 @@ func TestParser(t *testing.T) { "11926": true, "11927": true, }, - RemediatedBy: map[int]bool{ + RemediatedBy: map[uint]bool{ 5013943: true, }, }, @@ -887,7 +887,7 @@ func TestParser(t *testing.T) { ProductIDs: map[string]bool{ "10484": true, }, - RemediatedBy: map[int]bool{ + RemediatedBy: map[uint]bool{ 5014025: true, }, }, @@ -900,7 +900,7 @@ func TestParser(t *testing.T) { "10481": true, "10482": true, }, - RemediatedBy: map[int]bool{ + RemediatedBy: map[uint]bool{ 5014011: true, 5014001: true, }, @@ -909,7 +909,7 @@ func TestParser(t *testing.T) { } // A random vulnerability ("CVE-2022-29137") - expectedVendorFixes := map[string]map[int]parsed.VendorFix{ + expectedVendorFixes := map[string]map[uint]parsed.VendorFix{ "Windows 10": { 5013941: { FixedBuild: "10.0.17763.2928", @@ -918,7 +918,7 @@ func TestParser(t *testing.T) { "11569": true, "11570": true, }, - Supersedes: ptr.Int(5012647), + Supersedes: ptr.Uint(5012647), }, 5013952: { FixedBuild: "10.0.14393.5125", @@ -926,7 +926,7 @@ func TestParser(t *testing.T) { "10852": true, "10853": true, }, - Supersedes: ptr.Int(5012596), + Supersedes: ptr.Uint(5012596), }, 5013942: { FixedBuild: "10.0.19043.1706", @@ -941,7 +941,7 @@ func TestParser(t *testing.T) { "11930": true, "11931": true, }, - Supersedes: ptr.Int(5012599), + Supersedes: ptr.Uint(5012599), }, 5013963: { FixedBuild: "10.0.10240.19297", @@ -949,7 +949,7 @@ func TestParser(t *testing.T) { "10729": true, "10735": true, }, - Supersedes: ptr.Int(5012653), + Supersedes: ptr.Uint(5012653), }, 5013945: { @@ -959,7 +959,7 @@ func TestParser(t *testing.T) { "11713": true, "11714": true, }, - Supersedes: ptr.Int(5012591), + Supersedes: ptr.Uint(5012591), }, }, "Windows Server 2019": { @@ -969,7 +969,7 @@ func TestParser(t *testing.T) { "11571": true, "11572": true, }, - Supersedes: ptr.Int(5012647), + Supersedes: ptr.Uint(5012647), }, }, @@ -980,7 +980,7 @@ func TestParser(t *testing.T) { "11923": true, "11924": true, }, - Supersedes: ptr.Int(5012604), + Supersedes: ptr.Uint(5012604), }, }, @@ -990,149 +990,149 @@ func TestParser(t *testing.T) { ProductIDs: map[string]bool{ "11803": true, }, - Supersedes: ptr.Int(5012599), + Supersedes: ptr.Uint(5012599), }, }, "Windows Server 2008": { 5014010: { + FixedBuild: "6.0.6003.21481", ProductIDs: map[string]bool{ "9312": true, "10287": true, "9318": true, "9344": true, }, - FixedBuild: "6.0.6003.21481", - Supersedes: ptr.Int(5012658), + Supersedes: ptr.Uint(5012658), }, 5014006: { + FixedBuild: "6.0.6003.21481", ProductIDs: map[string]bool{ "9312": true, "10287": true, "9318": true, "9344": true, }, - FixedBuild: "6.0.6003.21481", }, }, "Windows Server 2008 R2": { 5014012: { + FixedBuild: "6.1.7601.25954", ProductIDs: map[string]bool{ "10051": true, "10049": true, }, - Supersedes: ptr.Int(5012626), - FixedBuild: "6.1.7601.25954", + Supersedes: ptr.Uint(5012626), }, 5013999: { + FixedBuild: "6.1.7601.25954", ProductIDs: map[string]bool{ "10051": true, "10049": true, }, - FixedBuild: "6.1.7601.25954", }, }, "Windows Server 2012": { 5014017: { + FixedBuild: "6.2.9200.23714", ProductIDs: map[string]bool{ "10378": true, "10379": true, }, - Supersedes: ptr.Int(5012650), - FixedBuild: "6.2.9200.23714", + Supersedes: ptr.Uint(5012650), }, 5014018: { + FixedBuild: "6.2.9200.23714", ProductIDs: map[string]bool{ "10378": true, "10379": true, }, - FixedBuild: "6.2.9200.23714", }, }, "Windows Server 2012 R2": { 5014011: { + FixedBuild: "6.3.9600.20371", ProductIDs: map[string]bool{ "10483": true, "10543": true, }, - FixedBuild: "6.3.9600.20371", - Supersedes: ptr.Int(5012670), + Supersedes: ptr.Uint(5012670), }, 5014001: { + FixedBuild: "6.3.9600.20365", ProductIDs: map[string]bool{ "10483": true, "10543": true, }, - FixedBuild: "6.3.9600.20365", }, }, "Windows 7": { 5014012: { + FixedBuild: "6.1.7601.25954", ProductIDs: map[string]bool{ "10047": true, "10048": true, }, - Supersedes: ptr.Int(5012626), - FixedBuild: "6.1.7601.25954", + Supersedes: ptr.Uint(5012626), }, 5013999: { + FixedBuild: "6.1.7601.25954", ProductIDs: map[string]bool{ "10047": true, "10048": true, }, - FixedBuild: "6.1.7601.25954", }, }, "Windows Server 2016": { 5013952: { + FixedBuild: "10.0.14393.5125", ProductIDs: map[string]bool{ "10816": true, "10855": true, }, - FixedBuild: "10.0.14393.5125", }, }, "Windows 11": { 5013943: { + FixedBuild: "10.0.22000.675", ProductIDs: map[string]bool{ "11926": true, "11927": true, }, - FixedBuild: "10.0.22000.675", - Supersedes: ptr.Int(5012592), + Supersedes: ptr.Uint(5012592), }, }, "Windows RT 8.1": { 5014025: { + FixedBuild: "6.3.9600.20367", ProductIDs: map[string]bool{ "10484": true, }, - FixedBuild: "6.3.9600.20367", }, }, "Windows 8.1": { 5014011: { + FixedBuild: "6.3.9600.20371", ProductIDs: map[string]bool{ "10481": true, "10482": true, }, - FixedBuild: "6.3.9600.20371", - Supersedes: ptr.Int(5012670), + Supersedes: ptr.Uint(5012670), }, 5014001: { + FixedBuild: "6.3.9600.20365", ProductIDs: map[string]bool{ "10481": true, "10482": true, }, - FixedBuild: "6.3.9600.20365", }, }, } @@ -1217,7 +1217,7 @@ func TestParser(t *testing.T) { for pID, pFn := range grp { expected = append( expected, - msrcxml.Product{ProductID: pID, FullName: pFn}, + msrcxml.Product{ProductID: pID, FullName: string(pFn)}, ) } } diff --git a/server/vulnerabilities/msrc/sync.go b/server/vulnerabilities/msrc/sync.go index 6df3ad6556..bf1c1301e3 100644 --- a/server/vulnerabilities/msrc/sync.go +++ b/server/vulnerabilities/msrc/sync.go @@ -1,6 +1,7 @@ package msrc import ( + "context" "fmt" "net/http" @@ -28,7 +29,7 @@ func bulletinsDelta( var matching []io.SecurityBulletinName for _, r := range remote { for _, o := range os { - product := parsed.NewProduct(o.Name) + product := parsed.NewProductFromOS(o) if r.ProductName() == product.Name() { matching = append(matching, r) } @@ -60,12 +61,12 @@ func bulletinsDelta( // Sync syncs the local msrc security bulletins (contained in dstDir) for one or more operating systems with the security // bulletin published in Github. // If 'os' is nil, then all security bulletins will be synched. -func Sync(client *http.Client, dstDir string, os []fleet.OperatingSystem) error { +func Sync(ctx context.Context, client *http.Client, dstDir string, os []fleet.OperatingSystem) error { rep := github.NewClient(client).Repositories gh := io.NewGitHubClient(client, rep, dstDir) fs := io.NewFSClient(dstDir) - if err := sync(os, fs, gh); err != nil { + if err := sync(ctx, os, fs, gh); err != nil { return fmt.Errorf("msrc sync: %w", err) } @@ -73,11 +74,12 @@ func Sync(client *http.Client, dstDir string, os []fleet.OperatingSystem) error } func sync( + ctx context.Context, os []fleet.OperatingSystem, fsClient io.FSAPI, ghClient io.GitHubAPI, ) error { - remoteURLs, err := ghClient.Bulletins() + remoteURLs, err := ghClient.Bulletins(ctx) if err != nil { return err } diff --git a/server/vulnerabilities/msrc/sync_test.go b/server/vulnerabilities/msrc/sync_test.go index 394737e2e4..43bb0b2ea6 100644 --- a/server/vulnerabilities/msrc/sync_test.go +++ b/server/vulnerabilities/msrc/sync_test.go @@ -1,6 +1,7 @@ package msrc import ( + "context" "testing" "github.com/fleetdm/fleet/v4/server/fleet" @@ -21,7 +22,7 @@ type testData struct { type ghMock struct{ testData *testData } -func (gh ghMock) Bulletins() (map[io.SecurityBulletinName]string, error) { +func (gh ghMock) Bulletins(ctx context.Context) (map[io.SecurityBulletinName]string, error) { return gh.testData.remoteList, gh.testData.remoteListError } @@ -42,6 +43,7 @@ func (fs fsMock) Delete(d io.SecurityBulletinName) error { } func TestSync(t *testing.T) { + ctx := context.Background() t.Run("#sync", func(t *testing.T) { os := []fleet.OperatingSystem{ { @@ -65,7 +67,7 @@ func TestSync(t *testing.T) { localList: []io.SecurityBulletinName{"Windows_10-2022_09_10.json"}, } - err := sync(os, fsMock{testData: &testData}, ghMock{testData: &testData}) + err := sync(ctx, os, fsMock{testData: &testData}, ghMock{testData: &testData}) require.NoError(t, err) require.ElementsMatch(t, testData.remoteDownloaded, []string{"http://somebulletin.com"}) require.ElementsMatch(t, testData.localDeleted, []io.SecurityBulletinName{"Windows_10-2022_09_10.json"}) diff --git a/server/vulnerabilities/nvd/cve.go b/server/vulnerabilities/nvd/cve.go index 5eba95fa14..0601839890 100644 --- a/server/vulnerabilities/nvd/cve.go +++ b/server/vulnerabilities/nvd/cve.go @@ -147,7 +147,7 @@ func TranslateCPEToCVE( var newVulns []fleet.SoftwareVulnerability for _, vuln := range vulns { - newCount, err := ds.InsertVulnerabilities(ctx, []fleet.SoftwareVulnerability{vuln}, fleet.NVDSource) + newCount, err := ds.InsertSoftwareVulnerabilities(ctx, []fleet.SoftwareVulnerability{vuln}, fleet.NVDSource) if err != nil { level.Error(logger).Log("cpe processing", "error", "err", err) continue diff --git a/server/vulnerabilities/nvd/cve_test.go b/server/vulnerabilities/nvd/cve_test.go index b83a754ca5..976dec753c 100644 --- a/server/vulnerabilities/nvd/cve_test.go +++ b/server/vulnerabilities/nvd/cve_test.go @@ -119,10 +119,10 @@ func (d *threadSafeDSMock) ListSoftwareCPEs(ctx context.Context) ([]fleet.Softwa return d.Store.ListSoftwareCPEs(ctx) } -func (d *threadSafeDSMock) InsertVulnerabilities(ctx context.Context, vulns []fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (int64, error) { +func (d *threadSafeDSMock) InsertSoftwareVulnerabilities(ctx context.Context, vulns []fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (int64, error) { d.mu.Lock() defer d.mu.Unlock() - return d.Store.InsertVulnerabilities(ctx, vulns, src) + return d.Store.InsertSoftwareVulnerabilities(ctx, vulns, src) } func TestTranslateCPEToCVE(t *testing.T) { @@ -149,7 +149,7 @@ func TestTranslateCPEToCVE(t *testing.T) { cveLock := &sync.Mutex{} var cvesFound []string - ds.InsertVulnerabilitiesFunc = func(ctx context.Context, vulns []fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (int64, error) { + ds.InsertSoftwareVulnerabilitiesFunc = func(ctx context.Context, vulns []fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (int64, error) { cveLock.Lock() defer cveLock.Unlock() for _, v := range vulns { @@ -180,7 +180,7 @@ func TestTranslateCPEToCVE(t *testing.T) { return softwareCPEs, nil } - ds.InsertVulnerabilitiesFunc = func(ctx context.Context, vulns []fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (int64, error) { + ds.InsertSoftwareVulnerabilitiesFunc = func(ctx context.Context, vulns []fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (int64, error) { return 1, nil } recent, err := TranslateCPEToCVE(ctx, safeDS, tempDir, kitlog.NewNopLogger(), true) @@ -188,7 +188,7 @@ func TestTranslateCPEToCVE(t *testing.T) { byCPE := make(map[uint]int) for _, cpe := range recent { - byCPE[cpe.SoftwareID]++ + byCPE[cpe.Affected()]++ } // even if it's somewhat far in the past, I've seen the exact numbers @@ -200,7 +200,7 @@ func TestTranslateCPEToCVE(t *testing.T) { // call it again but now return 0 from this call, simulating CVE-CPE pairs // that already existed in the DB. - ds.InsertVulnerabilitiesFunc = func(ctx context.Context, vulns []fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (int64, error) { + ds.InsertSoftwareVulnerabilitiesFunc = func(ctx context.Context, vulns []fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (int64, error) { return 0, nil } recent, err = TranslateCPEToCVE(ctx, safeDS, tempDir, kitlog.NewNopLogger(), true) diff --git a/server/vulnerabilities/oval/analyzer.go b/server/vulnerabilities/oval/analyzer.go index 5b999b0b96..e4d44e96f6 100644 --- a/server/vulnerabilities/oval/analyzer.go +++ b/server/vulnerabilities/oval/analyzer.go @@ -3,17 +3,13 @@ package oval import ( "context" "encoding/json" - "errors" "fmt" - "io/fs" "io/ioutil" - "os" - "path/filepath" - "strings" "time" "github.com/fleetdm/fleet/v4/server/fleet" oval_parsed "github.com/fleetdm/fleet/v4/server/vulnerabilities/oval/parsed" + utils "github.com/fleetdm/fleet/v4/server/vulnerabilities/utils" ) const ( @@ -85,7 +81,7 @@ func Analyze( } for _, hId := range hIds { - insrt, del := vulnsDelta(foundInBatch[hId], existingInBatch[hId]) + insrt, del := utils.VulnsDelta(foundInBatch[hId], existingInBatch[hId]) for _, i := range insrt { toInsertSet[i.Key()] = i } @@ -95,9 +91,9 @@ func Analyze( } } - err = batchProcess(toDeleteSet, func(v []fleet.SoftwareVulnerability) error { + err = utils.BatchProcess(toDeleteSet, func(v []fleet.SoftwareVulnerability) error { return ds.DeleteSoftwareVulnerabilities(ctx, v) - }) + }, vulnBatchSize) if err != nil { return nil, err } @@ -107,8 +103,8 @@ func Analyze( inserted = make([]fleet.SoftwareVulnerability, 0, len(toInsertSet)) } - err = batchProcess(toInsertSet, func(v []fleet.SoftwareVulnerability) error { - n, err := ds.InsertVulnerabilities(ctx, v, source) + err = utils.BatchProcess(toInsertSet, func(v []fleet.SoftwareVulnerability) error { + n, err := ds.InsertSoftwareVulnerabilities(ctx, v, source) if err != nil { return err } @@ -118,7 +114,7 @@ func Analyze( } return nil - }) + }, vulnBatchSize) if err != nil { return nil, err } @@ -126,79 +122,14 @@ func Analyze( return inserted, nil } -func batchProcess( - values map[string]fleet.SoftwareVulnerability, - dsFunc func(v []fleet.SoftwareVulnerability) error, -) error { - if len(values) == 0 { - return nil - } - - bSize := vulnBatchSize - if bSize > len(values) { - bSize = len(values) - } - - buffer := make([]fleet.SoftwareVulnerability, bSize) - var offset, i int - for _, v := range values { - buffer[offset] = v - offset++ - i++ - - // Consume buffer if full or if we are at the last iteration - if offset == bSize || i >= len(values) { - err := dsFunc(buffer[:offset]) - if err != nil { - return err - } - offset = 0 - } - } - return nil -} - -// vulnsDelta compares what vulnerabilities already exists with what new vulnerabilities were found -// and returns what to insert and what to delete. -func vulnsDelta( - found []fleet.SoftwareVulnerability, - existing []fleet.SoftwareVulnerability, -) (toInsert []fleet.SoftwareVulnerability, toDelete []fleet.SoftwareVulnerability) { - toDelete = make([]fleet.SoftwareVulnerability, 0) - toInsert = make([]fleet.SoftwareVulnerability, 0) - - existingSet := make(map[string]bool) - for _, e := range existing { - existingSet[e.Key()] = true - } - - foundSet := make(map[string]bool) - for _, f := range found { - foundSet[f.Key()] = true - } - - for _, e := range existing { - if _, ok := foundSet[e.Key()]; !ok { - toDelete = append(toDelete, e) - } - } - - for _, f := range found { - if _, ok := existingSet[f.Key()]; !ok { - toInsert = append(toInsert, f) - } - } - - return toInsert, toDelete -} - // loadDef returns the latest oval Definition for the given platform. func loadDef(platform Platform, vulnPath string) (oval_parsed.Result, error) { if !platform.IsSupported() { return nil, fmt.Errorf("platform %q not supported", platform) } - latest, err := latestOvalDefFor(platform, vulnPath, time.Now()) + fileName := platform.ToFilename(time.Now(), "json") + latest, err := utils.LatestFile(fileName, vulnPath) if err != nil { return nil, err } @@ -225,42 +156,3 @@ func loadDef(platform Platform, vulnPath string) (oval_parsed.Result, error) { return nil, fmt.Errorf("don't know how to parse file %q for %q platform", latest, platform) } - -// latestOvalDefFor returns the path of the OVAL definition for the given 'platform' in -// 'vulnPath' for the given 'date'. -// If not found, returns the most up to date OVAL definition for the given 'platform' -func latestOvalDefFor(platform Platform, vulnPath string, date time.Time) (string, error) { - ext := "json" - fileName := platform.ToFilename(date, ext) - target := filepath.Join(vulnPath, fileName) - - switch _, err := os.Stat(target); { - case err == nil: - return target, nil - case errors.Is(err, fs.ErrNotExist): - files, err := os.ReadDir(vulnPath) - if err != nil { - return "", err - } - - prefix := strings.Split(fileName, "-")[0] - var latest os.FileInfo - for _, f := range files { - if strings.HasPrefix(f.Name(), prefix) && strings.HasSuffix(f.Name(), ext) { - info, err := f.Info() - if err != nil { - continue - } - if latest == nil || info.ModTime().After(latest.ModTime()) { - latest = info - } - } - } - if latest == nil { - return "", fmt.Errorf("file not found for platform '%s' in '%s'", platform, vulnPath) - } - return filepath.Join(vulnPath, latest.Name()), nil - default: - return "", fmt.Errorf("failed to stat %q: %w", target, err) - } -} diff --git a/server/vulnerabilities/oval/analyzer_test.go b/server/vulnerabilities/oval/analyzer_test.go index 971a1c6982..f67cc30bae 100644 --- a/server/vulnerabilities/oval/analyzer_test.go +++ b/server/vulnerabilities/oval/analyzer_test.go @@ -363,82 +363,6 @@ func TestOvalAnalyzer(t *testing.T) { } }) - t.Run("#vulnsDelta", func(t *testing.T) { - t.Run("no existing vulnerabilities", func(t *testing.T) { - var found []fleet.SoftwareVulnerability - var existing []fleet.SoftwareVulnerability - - toInsert, toDelete := vulnsDelta(found, existing) - require.Empty(t, toInsert) - require.Empty(t, toDelete) - }) - - t.Run("existing match found", func(t *testing.T) { - found := []fleet.SoftwareVulnerability{ - {SoftwareID: 1, CVE: "cve_1"}, - {SoftwareID: 1, CVE: "cve_2"}, - {SoftwareID: 2, CVE: "cve_3"}, - {SoftwareID: 2, CVE: "cve_4"}, - } - - existing := []fleet.SoftwareVulnerability{ - {SoftwareID: 1, CVE: "cve_1"}, - {SoftwareID: 1, CVE: "cve_2"}, - {SoftwareID: 2, CVE: "cve_3"}, - {SoftwareID: 2, CVE: "cve_4"}, - } - - toInsert, toDelete := vulnsDelta(found, existing) - require.Empty(t, toInsert) - require.Empty(t, toDelete) - }) - - t.Run("existing differ from found", func(t *testing.T) { - found := []fleet.SoftwareVulnerability{ - {SoftwareID: 1, CVE: "cve_1"}, - {SoftwareID: 1, CVE: "cve_2"}, - {SoftwareID: 3, CVE: "cve_5"}, - {SoftwareID: 3, CVE: "cve_6"}, - } - - existing := []fleet.SoftwareVulnerability{ - {SoftwareID: 1, CVE: "cve_1"}, - {SoftwareID: 1, CVE: "cve_2"}, - {SoftwareID: 2, CVE: "cve_3"}, - {SoftwareID: 2, CVE: "cve_4"}, - } - - expectedToInsert := []fleet.SoftwareVulnerability{ - {SoftwareID: 3, CVE: "cve_5"}, - {SoftwareID: 3, CVE: "cve_6"}, - } - - expectedToDelete := []fleet.SoftwareVulnerability{ - {SoftwareID: 2, CVE: "cve_3"}, - {SoftwareID: 2, CVE: "cve_4"}, - } - - toInsert, toDelete := vulnsDelta(found, existing) - require.Equal(t, expectedToInsert, toInsert) - require.ElementsMatch(t, expectedToDelete, toDelete) - }) - - t.Run("nothing found but vulns exist", func(t *testing.T) { - var found []fleet.SoftwareVulnerability - - existing := []fleet.SoftwareVulnerability{ - {SoftwareID: 1, CVE: "cve_1"}, - {SoftwareID: 1, CVE: "cve_2"}, - {SoftwareID: 2, CVE: "cve_3"}, - {SoftwareID: 2, CVE: "cve_4"}, - } - - toInsert, toDelete := vulnsDelta(found, existing) - require.Empty(t, toInsert) - require.ElementsMatch(t, existing, toDelete) - }) - }) - t.Run("#load", func(t *testing.T) { t.Run("invalid vuln path", func(t *testing.T) { platform := NewPlatform("ubuntu", "Ubuntu 20.4.0") @@ -446,57 +370,4 @@ func TestOvalAnalyzer(t *testing.T) { require.Error(t, err, "invalid vulnerabity path") }) }) - - t.Run("#latestOvalDefFor", func(t *testing.T) { - t.Run("definition matching platform for date exists", func(t *testing.T) { - path := t.TempDir() - - today := time.Now() - platform := NewPlatform("ubuntu", "Ubuntu 20.4.0") - def := filepath.Join(path, platform.ToFilename(today, "json")) - - f1, err := os.Create(def) - require.NoError(t, err) - f1.Close() - - result, err := latestOvalDefFor(platform, path, today) - require.NoError(t, err) - require.Equal(t, def, result) - }) - - t.Run("definition matching platform exists but not for date", func(t *testing.T) { - path := t.TempDir() - - today := time.Now() - yesterday := today.Add(-24 * time.Hour) - - platform := NewPlatform("ubuntu", "Ubuntu 20.4.0") - def := filepath.Join(path, platform.ToFilename(yesterday, "json")) - - f1, err := os.Create(def) - require.NoError(t, err) - f1.Close() - - result, err := latestOvalDefFor(platform, path, today) - require.NoError(t, err) - require.Equal(t, def, result) - }) - - t.Run("definition does not exists for platform", func(t *testing.T) { - path := t.TempDir() - - today := time.Now() - - platform1 := NewPlatform("ubuntu", "Ubuntu 20.4.0") - def1 := filepath.Join(path, platform1.ToFilename(today, "json")) - f1, err := os.Create(def1) - require.NoError(t, err) - f1.Close() - - platform2 := NewPlatform("ubuntu", "Ubuntu 18.4.0") - - _, err = latestOvalDefFor(platform2, path, today) - require.Error(t, err, "file not found for platform") - }) - }) } diff --git a/server/vulnerabilities/oval/parsed/dpkg_infotest.go b/server/vulnerabilities/oval/parsed/dpkg_infotest.go index 55c538e02d..059322c8b1 100644 --- a/server/vulnerabilities/oval/parsed/dpkg_infotest.go +++ b/server/vulnerabilities/oval/parsed/dpkg_infotest.go @@ -1,6 +1,9 @@ package oval_parsed -import "github.com/fleetdm/fleet/v4/server/fleet" +import ( + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/vulnerabilities/utils" +) // DpkgInfoTest encapsulates a Dpkg info test. // see https://oval.mitre.org/language/version5.10.1/ovaldefinition/documentation/linux-definitions-schema.html#dpkginfo_test @@ -51,7 +54,7 @@ func (t *DpkgInfoTest) matches(software []fleet.Software) (int, int, []fleet.Sof r := make([]bool, 0) for _, s := range t.States { - evalR, err := s.Eval(p.Version, Rpmvercmp, false) + evalR, err := s.Eval(p.Version, utils.Rpmvercmp, false) if err != nil { return 0, 0, nil, err } diff --git a/server/vulnerabilities/oval/parsed/object_info_state.go b/server/vulnerabilities/oval/parsed/object_info_state.go index fc2457ac88..8717944b6a 100644 --- a/server/vulnerabilities/oval/parsed/object_info_state.go +++ b/server/vulnerabilities/oval/parsed/object_info_state.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/vulnerabilities/utils" ) type ObjectInfoState struct { @@ -58,7 +59,7 @@ func (sta ObjectInfoState) EvalSoftware(s fleet.Software) (bool, error) { rel = s.Release } else { // If not, try to get it from the version - rel = release(s.Version) + rel = utils.Release(s.Version) } rEval, err := sta.Release.Eval(rel) if err != nil { @@ -87,7 +88,7 @@ func (sta ObjectInfoState) EvalSoftware(s fleet.Software) (bool, error) { // TODO: see https://github.com/fleetdm/fleet/issues/6236 - // ATM we are not storing the epoch, so we will need to removed it from the // state ... otherwise we will - rEval, err := sta.Evr.Eval(evr, Rpmvercmp, true) + rEval, err := sta.Evr.Eval(evr, utils.Rpmvercmp, true) if err != nil { return false, err } diff --git a/server/vulnerabilities/oval/parsed/object_state_simple_value.go b/server/vulnerabilities/oval/parsed/object_state_simple_value.go index fdeb19a8c9..8472fee0b3 100644 --- a/server/vulnerabilities/oval/parsed/object_state_simple_value.go +++ b/server/vulnerabilities/oval/parsed/object_state_simple_value.go @@ -4,6 +4,8 @@ import ( "fmt" "strconv" "strings" + + "github.com/fleetdm/fleet/v4/server/vulnerabilities/utils" ) type ObjectStateSimpleValue string @@ -59,7 +61,7 @@ func (sta ObjectStateSimpleValue) Eval(other string) (bool, error) { } case EvrString: evr := NewObjectStateEvrString(op.String(), val) - return evr.Eval(other, Rpmvercmp, true) + return evr.Eval(other, utils.Rpmvercmp, true) case Float: val1, err := strconv.ParseFloat(val, 32) if err != nil { diff --git a/server/vulnerabilities/oval/parsed/rpmvercmp.go b/server/vulnerabilities/utils/rpmvercmp.go similarity index 98% rename from server/vulnerabilities/oval/parsed/rpmvercmp.go rename to server/vulnerabilities/utils/rpmvercmp.go index 115656e733..7c5016f1ca 100644 --- a/server/vulnerabilities/oval/parsed/rpmvercmp.go +++ b/server/vulnerabilities/utils/rpmvercmp.go @@ -1,4 +1,4 @@ -package oval_parsed +package utils import ( "strconv" @@ -33,7 +33,7 @@ func Rpmvercmp(a, b string) int { return r } - return rpmCmp(release(a), release(b)) + return rpmCmp(Release(a), Release(b)) } type segment struct { @@ -219,7 +219,7 @@ func rpmCmp(a, b string) int { return -1 } -func release(v string) string { +func Release(v string) string { var s int e := len(v) var seen bool diff --git a/server/vulnerabilities/oval/parsed/rpmvercmp_test.go b/server/vulnerabilities/utils/rpmvercmp_test.go similarity index 99% rename from server/vulnerabilities/oval/parsed/rpmvercmp_test.go rename to server/vulnerabilities/utils/rpmvercmp_test.go index 7c8800caee..d32e1a9bf8 100644 --- a/server/vulnerabilities/oval/parsed/rpmvercmp_test.go +++ b/server/vulnerabilities/utils/rpmvercmp_test.go @@ -1,4 +1,4 @@ -package oval_parsed +package utils import ( "testing" @@ -50,7 +50,7 @@ func TestVersionParts(t *testing.T) { for _, c := range cases { require.Equal(t, c.epoch, epoch(c.v)) require.Equal(t, c.version, version(c.v)) - require.Equal(t, c.release, release(c.v)) + require.Equal(t, c.release, Release(c.v)) } } @@ -4557,6 +4557,7 @@ func TestRpmvercmp(t *testing.T) { {"2~", GREATER, "~a"}, {"2~", GREATER, "1~"}, {"2~", EQUAL, "2~"}, + {"10.0.22000.795", LESS, "10.0.22000.796"}, } for _, c := range cases { diff --git a/server/vulnerabilities/utils/utils.go b/server/vulnerabilities/utils/utils.go new file mode 100644 index 0000000000..cc6713c3f5 --- /dev/null +++ b/server/vulnerabilities/utils/utils.go @@ -0,0 +1,162 @@ +package utils + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/fleetdm/fleet/v4/server/fleet" +) + +// RecentVulns filters vulnerabilities based on whether the vulnerability cve is contained in 'meta'. +// Returns the filtered vulnerabilities and their meta data. +func RecentVulns[T fleet.Vulnerability]( + vulns []T, + meta []fleet.CVEMeta, +) ([]T, map[string]fleet.CVEMeta) { + if len(vulns) == 0 { + return nil, nil + } + + recent := make(map[string]fleet.CVEMeta) + for _, r := range meta { + recent[r.CVE] = r + } + + seen := make(map[string]bool) + var r []T + + for _, v := range vulns { + if _, ok := recent[v.GetCVE()]; ok && !seen[v.Key()] { + seen[v.Key()] = true + r = append(r, v) + } + } + + return r, recent +} + +func BatchProcess[T fleet.Vulnerability]( + values map[string]T, + dsFunc func(v []T) error, + batchSize int, +) error { + if len(values) == 0 { + return nil + } + + bSize := batchSize + if bSize > len(values) { + bSize = len(values) + } + + buffer := make([]T, bSize) + var offset, i int + for _, v := range values { + buffer[offset] = v + offset++ + i++ + + // Consume buffer if full or if we are at the last iteration + if offset == bSize || i >= len(values) { + err := dsFunc(buffer[:offset]) + if err != nil { + return err + } + offset = 0 + } + } + return nil +} + +// VulnsDelta compares what vulnerabilities already exists with what new vulnerabilities were found +// and returns what to insert and what to delete. +func VulnsDelta[T fleet.Vulnerability]( + found []T, + existing []T, +) (toInsert []T, toDelete []T) { + toDelete = make([]T, 0) + toInsert = make([]T, 0) + + existingSet := make(map[string]bool) + for _, e := range existing { + existingSet[e.Key()] = true + } + + foundSet := make(map[string]bool) + for _, f := range found { + foundSet[f.Key()] = true + } + + for _, e := range existing { + if _, ok := foundSet[e.Key()]; !ok { + toDelete = append(toDelete, e) + } + } + + for _, f := range found { + if _, ok := existingSet[f.Key()]; !ok { + toInsert = append(toInsert, f) + } + } + + return toInsert, toDelete +} + +// ProductIDsIntersect given two sets of product IDs returns whether they have any elements in common +func ProductIDsIntersect(a map[string]bool, b map[string]bool) bool { + smallest := a + biggest := b + + if len(a) > len(b) { + smallest = b + biggest = a + } + + for pID := range smallest { + if biggest[pID] { + return true + } + } + return false +} + +// LatestFile returns the path of 'fileName' in 'dir' if the file exists, otherwise it will +// return the most recent file (based on the timestamp contained in 'fileName'). +func LatestFile(fileName string, dir string) (string, error) { + target := filepath.Join(dir, fileName) + ext := filepath.Ext(target) + + switch _, err := os.Stat(target); { + case err == nil: + return target, nil + case errors.Is(err, fs.ErrNotExist): + files, err := os.ReadDir(dir) + if err != nil { + return "", err + } + + prefix := strings.Split(fileName, "-")[0] + var latest os.FileInfo + for _, f := range files { + if strings.HasPrefix(f.Name(), prefix) && strings.HasSuffix(f.Name(), ext) { + info, err := f.Info() + if err != nil { + continue + } + if latest == nil || info.ModTime().After(latest.ModTime()) { + latest = info + } + } + } + if latest == nil { + return "", fmt.Errorf("file not found '%s' in '%s'", fileName, dir) + } + return filepath.Join(dir, latest.Name()), nil + default: + return "", fmt.Errorf("failed to stat %q: %w", target, err) + } +} diff --git a/server/vulnerabilities/utils/utils_test.go b/server/vulnerabilities/utils/utils_test.go new file mode 100644 index 0000000000..6a424ecaf7 --- /dev/null +++ b/server/vulnerabilities/utils/utils_test.go @@ -0,0 +1,226 @@ +package utils + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" +) + +func TestRecentVulns(t *testing.T) { + meta := []fleet.CVEMeta{ + {CVE: "cve-recent-1"}, + {CVE: "cve-recent-2"}, + {CVE: "cve-recent-3"}, + } + + t.Run("no NVD nor OVAL vulns", func(t *testing.T) { + vulns, meta := RecentVulns[fleet.SoftwareVulnerability](nil, meta) + require.Empty(t, vulns) + require.Empty(t, meta) + }) + + t.Run("filters vulnerabilities based on max age", func(t *testing.T) { + ovalVulns := []fleet.SoftwareVulnerability{ + {CVE: "cve-recent-1"}, + {CVE: "cve-recent-2"}, + {CVE: "cve-recent-2"}, + {CVE: "cve-outdated-1"}, + } + + nvdVulns := []fleet.SoftwareVulnerability{ + {CVE: "cve-recent-1"}, + {CVE: "cve-recent-3"}, + {CVE: "cve-outdated-2"}, + {CVE: "cve-outdated-3"}, + } + + expected := []string{ + "cve-recent-1", + "cve-recent-2", + "cve-recent-3", + } + + var input []fleet.SoftwareVulnerability + for _, e := range ovalVulns { + input = append(input, e) + } + for _, e := range nvdVulns { + input = append(input, e) + } + + var actual []string + vulns, meta := RecentVulns(input, meta) + for _, r := range vulns { + actual = append(actual, r.GetCVE()) + } + + expectedMeta := map[string]fleet.CVEMeta{ + "cve-recent-1": {CVE: "cve-recent-1"}, + "cve-recent-2": {CVE: "cve-recent-2"}, + "cve-recent-3": {CVE: "cve-recent-3"}, + } + + require.Equal(t, len(expected), len(actual)) + require.ElementsMatch(t, expected, actual) + require.Equal(t, expectedMeta, meta) + }) +} + +func TestVulnsDelta(t *testing.T) { + t.Run("no existing vulnerabilities", func(t *testing.T) { + var found []fleet.SoftwareVulnerability + var existing []fleet.SoftwareVulnerability + + toInsert, toDelete := VulnsDelta(found, existing) + require.Empty(t, toInsert) + require.Empty(t, toDelete) + }) + + t.Run("existing match found", func(t *testing.T) { + found := []fleet.SoftwareVulnerability{ + {SoftwareID: 1, CVE: "cve_1"}, + {SoftwareID: 1, CVE: "cve_2"}, + {SoftwareID: 2, CVE: "cve_3"}, + {SoftwareID: 2, CVE: "cve_4"}, + } + + existing := []fleet.SoftwareVulnerability{ + {SoftwareID: 1, CVE: "cve_1"}, + {SoftwareID: 1, CVE: "cve_2"}, + {SoftwareID: 2, CVE: "cve_3"}, + {SoftwareID: 2, CVE: "cve_4"}, + } + + toInsert, toDelete := VulnsDelta(found, existing) + require.Empty(t, toInsert) + require.Empty(t, toDelete) + }) + + t.Run("existing differ from found", func(t *testing.T) { + found := []fleet.SoftwareVulnerability{ + {SoftwareID: 1, CVE: "cve_1"}, + {SoftwareID: 1, CVE: "cve_2"}, + {SoftwareID: 3, CVE: "cve_5"}, + {SoftwareID: 3, CVE: "cve_6"}, + } + + existing := []fleet.SoftwareVulnerability{ + {SoftwareID: 1, CVE: "cve_1"}, + {SoftwareID: 1, CVE: "cve_2"}, + {SoftwareID: 2, CVE: "cve_3"}, + {SoftwareID: 2, CVE: "cve_4"}, + } + + expectedToInsert := []fleet.SoftwareVulnerability{ + {SoftwareID: 3, CVE: "cve_5"}, + {SoftwareID: 3, CVE: "cve_6"}, + } + + expectedToDelete := []fleet.SoftwareVulnerability{ + {SoftwareID: 2, CVE: "cve_3"}, + {SoftwareID: 2, CVE: "cve_4"}, + } + + toInsert, toDelete := VulnsDelta(found, existing) + require.Equal(t, expectedToInsert, toInsert) + require.ElementsMatch(t, expectedToDelete, toDelete) + }) + + t.Run("nothing found but vulns exist", func(t *testing.T) { + var found []fleet.SoftwareVulnerability + + existing := []fleet.SoftwareVulnerability{ + {SoftwareID: 1, CVE: "cve_1"}, + {SoftwareID: 1, CVE: "cve_2"}, + {SoftwareID: 2, CVE: "cve_3"}, + {SoftwareID: 2, CVE: "cve_4"}, + } + + toInsert, toDelete := VulnsDelta(found, existing) + require.Empty(t, toInsert) + require.ElementsMatch(t, existing, toDelete) + }) +} + +func TestProductsIntersect(t *testing.T) { + a := map[string]bool{ + "1": true, + "2": true, + "3": true, + } + + b := map[string]bool{ + "1": true, + } + + c := map[string]bool{ + "10": true, + } + + d := make(map[string]bool) + + require.True(t, ProductIDsIntersect(a, b)) + require.True(t, ProductIDsIntersect(b, a)) + + require.False(t, ProductIDsIntersect(b, c)) + require.False(t, ProductIDsIntersect(c, b)) + + require.False(t, ProductIDsIntersect(b, d)) + require.False(t, ProductIDsIntersect(d, b)) +} + +func TestLatestFile(t *testing.T) { + t.Run("file exists", func(t *testing.T) { + dir := t.TempDir() + + today := time.Now() + fileName := fmt.Sprintf("file1-%d_%02d_%02d.%s", today.Year(), today.Month(), today.Day(), "json") + + f1, err := os.Create(filepath.Join(dir, fileName)) + require.NoError(t, err) + f1.Close() + + result, err := LatestFile(fileName, dir) + require.NoError(t, err) + require.Equal(t, filepath.Join(dir, fileName), result) + }) + + t.Run("file exists but not for date", func(t *testing.T) { + dir := t.TempDir() + + today := time.Now() + yesterday := today.Add(-24 * time.Hour) + + todayFile := fmt.Sprintf("file1-%d_%02d_%02d.%s", today.Year(), today.Month(), today.Day(), "json") + yesterdayFile := fmt.Sprintf("file1-%d_%02d_%02d.%s", yesterday.Year(), yesterday.Month(), yesterday.Day(), "json") + + f1, err := os.Create(filepath.Join(dir, yesterdayFile)) + require.NoError(t, err) + f1.Close() + + result, err := LatestFile(todayFile, dir) + require.NoError(t, err) + require.Equal(t, filepath.Join(dir, yesterdayFile), result) + }) + + t.Run("file does not exists", func(t *testing.T) { + dir := t.TempDir() + + today := time.Now() + + wantedFile := fmt.Sprintf("file1-%d_%02d_%02d.%s", today.Year(), today.Month(), today.Day(), "json") + existingFile := fmt.Sprintf("file2-%d_%02d_%02d.%s", today.Year(), today.Month(), today.Day(), "json") + + f1, err := os.Create(filepath.Join(dir, existingFile)) + require.NoError(t, err) + f1.Close() + + _, err = LatestFile(wantedFile, dir) + require.Error(t, err, "file not found") + }) +} diff --git a/server/webhooks/vulnerabilities.go b/server/webhooks/vulnerabilities.go index 8a7f59ad70..0baa9c8acf 100644 --- a/server/webhooks/vulnerabilities.go +++ b/server/webhooks/vulnerabilities.go @@ -36,9 +36,10 @@ func TriggerVulnerabilitiesWebhook( targetURL := vulnConfig.DestinationURL batchSize := vulnConfig.HostBatchSize + // TODO JUAN: Handle OS Vulns groups := make(map[string][]uint) for _, v := range args.Vulnerablities { - groups[v.CVE] = append(groups[v.CVE], v.SoftwareID) + groups[v.GetCVE()] = append(groups[v.GetCVE()], v.Affected()) } for cve, sIDs := range groups { diff --git a/server/webhooks/vulnerabilities_test.go b/server/webhooks/vulnerabilities_test.go index 2d3641e097..205f46a8c6 100644 --- a/server/webhooks/vulnerabilities_test.go +++ b/server/webhooks/vulnerabilities_test.go @@ -117,35 +117,48 @@ func TestTriggerVulnerabilitiesWebhook(t *testing.T) { }, { "1 vuln in multiple software, 1 host", - []fleet.SoftwareVulnerability{{CVE: cves[0], SoftwareID: 1}, {CVE: cves[0], SoftwareID: 1}, {CVE: cves[0], SoftwareID: 2}}, + []fleet.SoftwareVulnerability{ + {CVE: cves[0], SoftwareID: 1}, + {CVE: cves[0], SoftwareID: 1}, + {CVE: cves[0], SoftwareID: 2}, + }, nil, hosts[:1], fmt.Sprintf("%s[%s]}}", jsonCVE1, jsonH1), }, { "1 vuln, 2 hosts", - []fleet.SoftwareVulnerability{{CVE: cves[0], SoftwareID: 1}}, + []fleet.SoftwareVulnerability{ + {CVE: cves[0], SoftwareID: 1}, + }, nil, hosts[:2], fmt.Sprintf("%s[%s,%s]}}", jsonCVE1, jsonH1, jsonH2), }, { "1 vuln, 3 hosts", - []fleet.SoftwareVulnerability{{CVE: cves[0], SoftwareID: 1}}, + []fleet.SoftwareVulnerability{ + {CVE: cves[0], SoftwareID: 1}, + }, nil, hosts[:3], fmt.Sprintf("%s[%s,%s]}}\n%s[%s]}}", jsonCVE1, jsonH1, jsonH2, jsonCVE1, jsonH3), // 2 requests, batch of 2 max }, { "1 vuln, 4 hosts", - []fleet.SoftwareVulnerability{{CVE: cves[0], SoftwareID: 1}}, + []fleet.SoftwareVulnerability{ + {CVE: cves[0], SoftwareID: 1}, + }, nil, hosts[:4], fmt.Sprintf("%s[%s,%s]}}\n%s[%s,%s]}}", jsonCVE1, jsonH1, jsonH2, jsonCVE1, jsonH3, jsonH4), // 2 requests, batch of 2 max }, { "2 vulns, 1 host each", - []fleet.SoftwareVulnerability{{CVE: cves[0], SoftwareID: 1}, {CVE: cves[1], SoftwareID: 2}}, + []fleet.SoftwareVulnerability{ + {CVE: cves[0], SoftwareID: 1}, + {CVE: cves[1], SoftwareID: 2}, + }, nil, hosts[:1], fmt.Sprintf("%s[%s]}}\n%s[%s]}}", jsonCVE1, jsonH1, jsonCVE2, jsonH1), diff --git a/server/worker/jira.go b/server/worker/jira.go index 5ad5ab41e9..1d637e7093 100644 --- a/server/worker/jira.go +++ b/server/worker/jira.go @@ -376,14 +376,14 @@ func QueueJiraVulnJobs( // _before_ we start processing them). cves := make([]string, 0, len(recentVulns)) for _, vuln := range recentVulns { - cves = append(cves, vuln.CVE) + cves = append(cves, vuln.GetCVE()) } sort.Strings(cves) level.Debug(logger).Log("recent_cves", fmt.Sprintf("%v", cves)) uniqCVEs := make(map[string]bool) for _, v := range recentVulns { - uniqCVEs[v.CVE] = true + uniqCVEs[v.GetCVE()] = true } for cve := range uniqCVEs { diff --git a/server/worker/zendesk.go b/server/worker/zendesk.go index be65e1e3c4..240e21166c 100644 --- a/server/worker/zendesk.go +++ b/server/worker/zendesk.go @@ -371,14 +371,14 @@ func QueueZendeskVulnJobs( // _before_ we start processing them). cves := make([]string, 0, len(recentVulns)) for _, vuln := range recentVulns { - cves = append(cves, vuln.CVE) + cves = append(cves, vuln.GetCVE()) } sort.Strings(cves) level.Debug(logger).Log("recent_cves", fmt.Sprintf("%v", cves)) uniqCVEs := make(map[string]bool) for _, v := range recentVulns { - uniqCVEs[v.CVE] = true + uniqCVEs[v.GetCVE()] = true } for cve := range uniqCVEs {