diff --git a/changes/bug-handle-missing-nvd-assets b/changes/bug-handle-missing-nvd-assets new file mode 100644 index 0000000000..2333348380 --- /dev/null +++ b/changes/bug-handle-missing-nvd-assets @@ -0,0 +1 @@ +Fall back to a previous NVD release if the asset we want is not in the latest release. \ No newline at end of file diff --git a/cmd/fleet/serve_test.go b/cmd/fleet/serve_test.go index c1fdbbfab3..0c80abcdda 100644 --- a/cmd/fleet/serve_test.go +++ b/cmd/fleet/serve_test.go @@ -487,12 +487,13 @@ func TestScanVulnerabilities(t *testing.T) { }, nil } - vulnPath := t.TempDir() + vulnPath := filepath.Join("..", "..", "server", "vulnerabilities", "testdata") config := config.VulnerabilitiesConfig{ DatabasesPath: vulnPath, Periodicity: 10 * time.Second, CurrentInstanceChecks: "auto", + DisableDataSync: true, } ctx = license.NewContext(ctx, &fleet.LicenseInfo{Tier: fleet.TierPremium}) diff --git a/pkg/download/download.go b/pkg/download/download.go index 2ce1d3b797..fab28572c9 100644 --- a/pkg/download/download.go +++ b/pkg/download/download.go @@ -28,7 +28,6 @@ func DownloadAndExtract(client *http.Client, u *url.URL, path string) error { } func download(client *http.Client, u *url.URL, path string, extract bool) error { - // atomically write to file dir, file := filepath.Split(path) if dir == "" { diff --git a/server/vulnerabilities/nvd/cpe.go b/server/vulnerabilities/nvd/cpe.go index 28ae9da80d..912c7c3fcf 100644 --- a/server/vulnerabilities/nvd/cpe.go +++ b/server/vulnerabilities/nvd/cpe.go @@ -10,6 +10,7 @@ import ( "path/filepath" "regexp" "strings" + "time" "github.com/doug-martin/goqu/v9" "github.com/fleetdm/fleet/v4/pkg/download" @@ -31,23 +32,36 @@ const ( var cpeDBRegex = regexp.MustCompile(`^cpe-.*\.sqlite\.gz$`) -func GetLatestGithubNVDRelease() (*github.RepositoryRelease, error) { - githubClient := github.NewClient(fleethttp.NewGithubClient()) - releases, _, err := githubClient.Repositories.ListReleases( - context.Background(), owner, repo, &github.ListOptions{Page: 0, PerPage: 10}, +// GetGithubNVDAsset looks at the last 10 releases and returns the first (release, asset) pair that +// matches pred +func GetGithubNVDAsset(pred func(rel *github.ReleaseAsset) bool) (*github.RepositoryRelease, *github.ReleaseAsset, error) { + ghClient := github.NewClient(fleethttp.NewGithubClient()) + + releases, _, err := ghClient.Repositories.ListReleases( + context.Background(), + owner, + repo, + &github.ListOptions{Page: 0, PerPage: 10}, ) if err != nil { - return nil, err + return nil, nil, err } for _, release := range releases { // skip draft releases - if !release.GetDraft() { - return release, nil + if release.GetDraft() { + continue } + + for _, asset := range release.Assets { + if pred(asset) { + return release, asset, nil + } + } + } - return nil, errors.New("no nvd release found") + return nil, nil, errors.New("no nvd release found") } // DownloadCPEDB downloads the CPE database to the given vulnPath. If cpeDBURL is empty, attempts to download it @@ -56,32 +70,34 @@ func DownloadCPEDBFromGithub(vulnPath string, cpeDBURL string) error { path := filepath.Join(vulnPath, cpeDBFilename) if cpeDBURL == "" { - release, err := GetLatestGithubNVDRelease() - if err != nil { - return err - } stat, err := os.Stat(path) switch { case errors.Is(err, os.ErrNotExist): // okay case err != nil: return err - default: - if stat.ModTime().After(release.CreatedAt.Time) { - // file is newer than release, do nothing - return nil - } + case stat.ModTime().Truncate(24 * time.Hour).Equal(time.Now().Truncate(24 * time.Hour)): + // Vulnerability assets are published once per day - if the asset in question has a + // mod date of 'today', then we can assume that is already up to day so there's nothing + // else to do. + return nil } - for _, asset := range release.Assets { - if cpeDBRegex.MatchString(asset.GetName()) { - cpeDBURL = asset.GetBrowserDownloadURL() - break - } + rel, asset, err := GetGithubNVDAsset(func(asset *github.ReleaseAsset) bool { + return cpeDBRegex.MatchString(asset.GetName()) + }) + if err != nil { + return err } - if cpeDBURL == "" { + if asset == nil { return errors.New("failed to find cpe database in nvd release") } + if stat != nil && stat.ModTime().After(rel.CreatedAt.Time) { + // file is newer than release, do nothing + return nil + } + + cpeDBURL = asset.GetBrowserDownloadURL() } u, err := url.Parse(cpeDBURL) diff --git a/server/vulnerabilities/nvd/cpe_translations.go b/server/vulnerabilities/nvd/cpe_translations.go index cfe0731117..ee95743b4a 100644 --- a/server/vulnerabilities/nvd/cpe_translations.go +++ b/server/vulnerabilities/nvd/cpe_translations.go @@ -8,10 +8,12 @@ import ( "os" "path/filepath" "regexp" + "time" "github.com/fleetdm/fleet/v4/pkg/download" "github.com/fleetdm/fleet/v4/pkg/fleethttp" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/google/go-github/v37/github" ) const cpeTranslationsFilename = "cpe_translations.json" @@ -37,32 +39,32 @@ func DownloadCPETranslationsFromGithub(vulnPath string, cpeTranslationsURL strin path := filepath.Join(vulnPath, cpeTranslationsFilename) if cpeTranslationsURL == "" { - release, err := GetLatestGithubNVDRelease() - if err != nil { - return err - } stat, err := os.Stat(path) switch { case errors.Is(err, os.ErrNotExist): // okay case err != nil: return err - default: - if stat.ModTime().After(release.CreatedAt.Time) { - // file is newer than release, do nothing - return nil - } + case stat.ModTime().Truncate(24 * time.Hour).Equal(time.Now().Truncate(24 * time.Hour)): + // Vulnerability assets are published once per day - if the asset in question has a + // mod date of 'today', then we can assume that is already up to day. + return nil } - for _, asset := range release.Assets { - if cpeTranslationsFilename == asset.GetName() { - cpeTranslationsURL = asset.GetBrowserDownloadURL() - break - } + release, asset, err := GetGithubNVDAsset(func(asset *github.ReleaseAsset) bool { + return cpeTranslationsFilename == asset.GetName() + }) + if err != nil { + return err } - if cpeTranslationsURL == "" { + if asset == nil { return errors.New("failed to find cpe translations in nvd release") } + if stat != nil && stat.ModTime().After(release.CreatedAt.Time) { + // file is newer than release, do nothing + return nil + } + cpeTranslationsURL = asset.GetBrowserDownloadURL() } u, err := url.Parse(cpeTranslationsURL) diff --git a/server/vulnerabilities/nvd/sync_test.go b/server/vulnerabilities/nvd/sync_test.go index 570dbe5bcd..80fa0e085b 100644 --- a/server/vulnerabilities/nvd/sync_test.go +++ b/server/vulnerabilities/nvd/sync_test.go @@ -2,11 +2,12 @@ package nvd import ( "context" - "github.com/fleetdm/fleet/v4/server/contexts/license" "path/filepath" "strings" "testing" + "github.com/fleetdm/fleet/v4/server/contexts/license" + "github.com/fleetdm/fleet/v4/pkg/nettest" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/mock" @@ -65,7 +66,7 @@ func TestLoadCVEMeta(t *testing.T) { require.Equal(t, false, *meta.CISAKnownExploit) meta = metaMap["CVE-2022-22587"] - require.Equal(t, (*float64)(nil), meta.CVSSScore) + require.Equal(t, float64(9.8), *meta.CVSSScore) require.Equal(t, float64(0.01843), *meta.EPSSProbability) require.Equal(t, true, *meta.CISAKnownExploit) } diff --git a/server/vulnerabilities/testdata/cpe.sqlite b/server/vulnerabilities/testdata/cpe.sqlite new file mode 100644 index 0000000000..e69de29bb2 diff --git a/server/vulnerabilities/testdata/nvdcve-1.1-2022.json.gz b/server/vulnerabilities/testdata/nvdcve-1.1-2022.json.gz new file mode 100644 index 0000000000..59fba0ecc2 Binary files /dev/null and b/server/vulnerabilities/testdata/nvdcve-1.1-2022.json.gz differ