diff --git a/changes/feature-7394-generate-msrc-artifacts b/changes/feature-7394-generate-msrc-artifacts new file mode 100644 index 0000000000..827aa8931e --- /dev/null +++ b/changes/feature-7394-generate-msrc-artifacts @@ -0,0 +1 @@ +Use MSRC parser to generate security bulletin artifacts to be used for detecting Windows OS vulnerabilities. \ No newline at end of file diff --git a/cmd/msrc/generate.go b/cmd/msrc/generate.go new file mode 100644 index 0000000000..4204d35555 --- /dev/null +++ b/cmd/msrc/generate.go @@ -0,0 +1,147 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "time" + + "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc" + "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/io" + "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/parsed" + "github.com/google/go-github/v37/github" +) + +func panicif(err error) { + if err != nil { + panic(err) + } +} + +func main() { + wd, err := os.Getwd() + panicif(err) + + now := time.Now() + httpC := http.DefaultClient + + ghAPI := io.NewGitHubClient(httpC, github.NewClient(httpC).Repositories, wd) + msrcAPI := io.NewMSRCClient(httpC, wd, io.MSRCBaseURL) + + fmt.Println("Downloading existing bulletins...") + eBulletins, err := ghAPI.Bulletins() + panicif(err) + + var bulletins []*parsed.SecurityBulletin + if len(eBulletins) == 0 { + fmt.Println("None found, backfilling...") + bulletins, err = backfill(now.Month(), now.Year(), msrcAPI) + panicif(err) + } else { + fmt.Println("Updating existing bulletins") + bulletins, err = update(now.Month(), now.Year(), eBulletins, msrcAPI, ghAPI) + panicif(err) + } + + fmt.Println("Saving bulletins...") + for _, b := range bulletins { + err := serialize(b, now, wd) + panicif(err) + } + + fmt.Println("Done.") +} + +func update( + m time.Month, + y int, + eBulletins map[io.SecurityBulletinName]string, + msrcClient io.MSRCAPI, + ghClient io.GitHubAPI, +) ([]*parsed.SecurityBulletin, error) { + fmt.Println("Downloading current feed...") + f, err := msrcClient.GetFeed(m, y) + if err != nil { + return nil, err + } + + fmt.Println("Parsing current feed...") + nBulletins, err := msrc.ParseFeed(f) + if err != nil { + return nil, err + } + + var bulletins []*parsed.SecurityBulletin + for _, url := range eBulletins { + fPath, err := ghClient.Download(url) + if err != nil { + return nil, err + } + + eB, err := parsed.UnmarshalBulletin(fPath) + if err != nil { + return nil, err + } + + nB, ok := nBulletins[eB.ProductName] + if ok { + if err = eB.Merge(nB); err != nil { + return nil, err + } + } + + bulletins = append(bulletins, eB) + } + + return bulletins, nil +} + +func backfill(upToM time.Month, upToY int, client io.MSRCAPI) ([]*parsed.SecurityBulletin, error) { + from := time.Date(io.MSRCMinYear, 1, 1, 0, 0, 0, 0, time.UTC) + upTo := time.Date(upToY, upToM+1, 1, 0, 0, 0, 0, time.UTC) + + bulletins := make(map[string]*parsed.SecurityBulletin) + for d := from; d.Before(upTo); d = d.AddDate(0, 1, 0) { + + fmt.Printf("Downloading feed for %d-%d...\n", d.Year(), d.Month()) + f, err := client.GetFeed(d.Month(), d.Year()) + if err != nil { + return nil, err + } + + fmt.Printf("Parsing feed for %d-%d...\n", d.Year(), d.Month()) + r, err := msrc.ParseFeed(f) + if err != nil { + return nil, err + } + + for name, nB := range r { + eB, ok := bulletins[name] + if !ok { + bulletins[name] = nB + continue + } + + if err = eB.Merge(nB); err != nil { + return nil, err + } + } + } + + var r []*parsed.SecurityBulletin + for _, b := range bulletins { + r = append(r, b) + } + + return r, nil +} + +func serialize(b *parsed.SecurityBulletin, d time.Time, wd string) error { + payload, err := json.Marshal(b) + if err != nil { + return err + } + filePath := io.FileName(b.ProductName, d, "json") + return os.WriteFile(filePath, payload, 0o644) +} diff --git a/server/vulnerabilities/msrc/io/fs.go b/server/vulnerabilities/msrc/io/fs.go index 5ade017705..42aafc63fc 100644 --- a/server/vulnerabilities/msrc/io/fs.go +++ b/server/vulnerabilities/msrc/io/fs.go @@ -6,29 +6,29 @@ import ( "strings" ) -type MSRCFSAPI interface { +type FSAPI interface { Bulletins() ([]SecurityBulletinName, error) Delete(SecurityBulletinName) error } -type MSRCFSClient struct { +type FSClient struct { dir string } -func NewMSRCFSClient(dir string) MSRCFSClient { - return MSRCFSClient{ +func NewFSClient(dir string) FSClient { + return FSClient{ dir: dir, } } // Delete deletes the provided security bulletin name from 'dir'. -func (fs MSRCFSClient) Delete(b SecurityBulletinName) error { +func (fs FSClient) Delete(b SecurityBulletinName) error { path := filepath.Join(fs.dir, string(b)) return os.Remove(path) } // Bulletins walks 'dir' returning all security bulletin names. -func (fs MSRCFSClient) Bulletins() ([]SecurityBulletinName, error) { +func (fs FSClient) Bulletins() ([]SecurityBulletinName, error) { var result []SecurityBulletinName err := filepath.WalkDir(fs.dir, func(path string, d os.DirEntry, err error) error { @@ -37,7 +37,7 @@ func (fs MSRCFSClient) Bulletins() ([]SecurityBulletinName, error) { } filePath := filepath.Base(path) - if strings.HasPrefix(filePath, MSRCFilePrefix) { + if strings.HasPrefix(filePath, mSRCFilePrefix) { result = append(result, NewSecurityBulletinName(filePath)) } diff --git a/server/vulnerabilities/msrc/io/fs_test.go b/server/vulnerabilities/msrc/io/fs_test.go index 9d8c8660da..e6ee060685 100644 --- a/server/vulnerabilities/msrc/io/fs_test.go +++ b/server/vulnerabilities/msrc/io/fs_test.go @@ -9,21 +9,21 @@ import ( "github.com/stretchr/testify/require" ) -func TestMSRCFSClient(t *testing.T) { +func TestFSClient(t *testing.T) { t.Run("#Bulletins", func(t *testing.T) { t.Run("directory does not exists", func(t *testing.T) { - sut := NewMSRCFSClient("asdf") + sut := NewFSClient("asdf") _, err := sut.Bulletins() require.Error(t, err) }) t.Run("returns a list of file matching the MSRC file prefix", func(t *testing.T) { path := t.TempDir() - sut := NewMSRCFSClient(path) + sut := NewFSClient(path) file1 := filepath.Join(path, "my_lyrics.json") - bulletin1 := filepath.Join(path, fmt.Sprintf("%sWindows_10-2022_10_10.json", MSRCFilePrefix)) - bulletin2 := filepath.Join(path, fmt.Sprintf("%sWindows_11-2022_10_10.json", MSRCFilePrefix)) + bulletin1 := filepath.Join(path, fmt.Sprintf("%sWindows_10-2022_10_10.json", mSRCFilePrefix)) + bulletin2 := filepath.Join(path, fmt.Sprintf("%sWindows_11-2022_10_10.json", mSRCFilePrefix)) f1, err := os.Create(bulletin1) require.NoError(t, err) diff --git a/server/vulnerabilities/msrc/io/github.go b/server/vulnerabilities/msrc/io/github.go index e58437e87f..1e6611e7ce 100644 --- a/server/vulnerabilities/msrc/io/github.go +++ b/server/vulnerabilities/msrc/io/github.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/url" + "path" "path/filepath" "strings" "time" @@ -13,36 +14,60 @@ import ( "github.com/google/go-github/v37/github" ) -type MSRCGithubAPI interface { - Download(SecurityBulletinName, string) error +// ReleaseLister interface around github.NewClient(...).Repositories. +type ReleaseLister interface { + ListReleases( + context.Context, + string, + string, + *github.ListOptions, + ) ([]*github.RepositoryRelease, *github.Response, error) +} + +// GitHubAPI allows users to interact with the MSRC artifacts published on Github. +type GitHubAPI interface { + Download(string) (string, error) Bulletins() (map[SecurityBulletinName]string, error) } -type MSRCGithubClient struct { - client *http.Client - dstDir string +type GitHubClient struct { + httpClient *http.Client + releases ReleaseLister + workDir string } -func NewMSRCGithubClient(client *http.Client, dir string) MSRCGithubClient { - return MSRCGithubClient{client: client, dstDir: dir} -} - -// Downloads the security bulletin to 'dir'. -func (gh MSRCGithubClient) Download(b SecurityBulletinName, urlStr string) error { - u, err := url.Parse(urlStr) - if err != nil { - return err +// NewGitHubClient returns a new GithubClient, 'workDir' will be used as the destination directory for +// downloading artifacts. +func NewGitHubClient(client *http.Client, releases ReleaseLister, workDir string) GitHubClient { + return GitHubClient{ + httpClient: client, + releases: releases, + workDir: workDir, } - path := filepath.Join(gh.dstDir, string(b)) - return download.DownloadAndExtract(gh.client, u, path) } -// Bulletins returns a map of 'name' => 'download URL' of the parsed security bulletins stored as assets on Github. -func (gh MSRCGithubClient) Bulletins() (map[SecurityBulletinName]string, error) { +// Download downloads the security bulletin located at 'URL' in 'workDir', returns the path of +// the downloaded bulletin. +func (gh GitHubClient) Download(URL string) (string, error) { + u, err := url.Parse(URL) + if err != nil { + return "", err + } + + fPath := filepath.Join(gh.workDir, path.Base(u.Path)) + if err := download.Download(gh.httpClient, u, fPath); err != nil { + return "", err + } + + return fPath, nil +} + +// Bulletins returns a map of 'bulletin name' => 'download URL' of the bulletins stored as assets on Github. +func (gh GitHubClient) Bulletins() (map[SecurityBulletinName]string, error) { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() - releases, r, err := github.NewClient(gh.client).Repositories.ListReleases( + releases, r, err := gh.releases.ListReleases( ctx, "fleetdm", "nvd", @@ -58,11 +83,9 @@ func (gh MSRCGithubClient) Bulletins() (map[SecurityBulletinName]string, error) results := make(map[SecurityBulletinName]string) - // TODO (juan): Since the nvd repo includes both NVD and MSRC assets, we will need to do some - // filtering logic here. To be done in https://github.com/fleetdm/fleet/issues/7394. for _, e := range releases[0].Assets { name := e.GetName() - if strings.HasPrefix(name, MSRCFilePrefix) { + if strings.HasPrefix(name, mSRCFilePrefix) { results[NewSecurityBulletinName(name)] = e.GetBrowserDownloadURL() } } diff --git a/server/vulnerabilities/msrc/io/github_test.go b/server/vulnerabilities/msrc/io/github_test.go new file mode 100644 index 0000000000..79aa53c160 --- /dev/null +++ b/server/vulnerabilities/msrc/io/github_test.go @@ -0,0 +1,115 @@ +package io + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/google/go-github/v37/github" + "github.com/stretchr/testify/require" +) + +type mockGHReleaseLister struct{} + +func (m mockGHReleaseLister) ListReleases( + ctx context.Context, + owner string, + repo string, + opts *github.ListOptions, +) ([]*github.RepositoryRelease, *github.Response, error) { + var releases []*github.RepositoryRelease + releases = append(releases, &github.RepositoryRelease{ + Assets: []*github.ReleaseAsset{ + { + ID: ptr.Int64(76142088), + URL: ptr.String("https://api.github.com/repos/fleetdm/nvd/releases/assets/76142088"), + Name: ptr.String("cpe-80f8ec9cfb9d810.sqlite.gz"), + Label: ptr.String(""), + State: ptr.String("uploaded"), + ContentType: ptr.String("application/gzip"), + Size: ptr.Int(52107588), + DownloadCount: ptr.Int(683), + BrowserDownloadURL: ptr.String("https://github.com/fleetdm/nvd/releases/download/202208290017/cpe-80f8ec9cfb9d810.sqlite"), + NodeID: ptr.String("RA_kwDOF19pRs4EidYI"), + }, + { + ID: ptr.Int64(76142089), + URL: ptr.String("https://api.github.com/repos/fleetdm/nvd/releases/assets/76142089"), + Name: ptr.String(fmt.Sprintf("%sWindows_10-2022_09_10.json", mSRCFilePrefix)), + Label: ptr.String(""), + State: ptr.String("uploaded"), + ContentType: ptr.String("application/json"), + Size: ptr.Int(52107588), + DownloadCount: ptr.Int(683), + BrowserDownloadURL: ptr.String(fmt.Sprintf("https://github.com/fleetdm/nvd/releases/download/202208290017/%sWindows_10-2022_09_10.json", mSRCFilePrefix)), + NodeID: ptr.String("RA_kwDOF19pRs4EidYA"), + }, + { + ID: ptr.Int64(76142090), + URL: ptr.String("https://api.github.com/repos/fleetdm/nvd/releases/assets/76142089"), + Name: ptr.String(fmt.Sprintf("%sWindows_11-2022_09_10.json", mSRCFilePrefix)), + Label: ptr.String(""), + State: ptr.String("uploaded"), + ContentType: ptr.String("application/json"), + Size: ptr.Int(52107588), + DownloadCount: ptr.Int(683), + BrowserDownloadURL: ptr.String(fmt.Sprintf("https://github.com/fleetdm/nvd/releases/download/202208290017/%sWindows_11-2022_09_10.json", mSRCFilePrefix)), + NodeID: ptr.String("RA_kwDOF19pRs4EidYA"), + }, + }, + }) + + res := &github.Response{ + Response: &http.Response{ + StatusCode: http.StatusOK, + }, + } + return releases, res, nil +} + +func TestGithubClient(t *testing.T) { + t.Run("#Download", func(t *testing.T) { + fileName := fmt.Sprintf("%sWindows_11-2022_09_10.json", mSRCFilePrefix) + urlPath := fmt.Sprintf("/fleetdm/nvd/releases/download/202208290017/%s", fileName) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == urlPath { + w.Header().Add("content-type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte("some payload")) + } + })) + t.Cleanup(server.Close) + + dstDir := t.TempDir() + expectedPath := filepath.Join(dstDir, fileName) + url := server.URL + urlPath + + sut := NewGitHubClient(server.Client(), mockGHReleaseLister{}, dstDir) + actualPath, err := sut.Download(url) + require.NoError(t, err) + require.Equal(t, expectedPath, actualPath) + require.FileExists(t, expectedPath) + }) + + t.Run("#Bulletins", func(t *testing.T) { + sut := NewGitHubClient(nil, mockGHReleaseLister{}, t.TempDir()) + + bulletins, err := sut.Bulletins() + require.NoError(t, err) + require.Len(t, bulletins, 2) + + expectedBulletins := []SecurityBulletinName{ + NewSecurityBulletinName(fmt.Sprintf("%sWindows_10-2022_09_10.json", mSRCFilePrefix)), + NewSecurityBulletinName(fmt.Sprintf("%sWindows_11-2022_09_10.json", mSRCFilePrefix)), + } + + for _, e := range expectedBulletins { + require.NotEmpty(t, bulletins[e]) + } + }) +} diff --git a/server/vulnerabilities/msrc/io/msrc.go b/server/vulnerabilities/msrc/io/msrc.go new file mode 100644 index 0000000000..8e967fe072 --- /dev/null +++ b/server/vulnerabilities/msrc/io/msrc.go @@ -0,0 +1,75 @@ +package io + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "path/filepath" + "time" + + "github.com/fleetdm/fleet/v4/pkg/download" +) + +const ( + // Pre 2020 there are some weirdness around the way the 'Supersedes' field is defined for Vulnerabilities, sometimes + // it does not reference a KBID. + MSRCMinYear = 2020 + MSRCBaseURL = `https://api.msrc.microsoft.com` +) + +// MSRCAPI allows users to interact with MSRC resources +type MSRCAPI interface { + GetFeed(time.Month, int) (string, error) +} + +type MSRCClient struct { + client *http.Client + workDir string + baseURL string +} + +// NewMSRCClient returns a new MSRCClient that will store all downloaded files in 'workDir' and will +// use 'baseURL' for doing http requests. +func NewMSRCClient( + client *http.Client, + workDir string, + baseURL string, +) MSRCClient { + return MSRCClient{client: client, workDir: workDir, baseURL: baseURL} +} + +func feedName(date time.Time) string { + return date.Format("2006-Jan") +} + +func (msrc MSRCClient) getURL(date time.Time) (*url.URL, error) { + return url.Parse(msrc.baseURL + "/cvrf/v2.0/document/" + feedName(date)) +} + +// GetFeed downloads the MSRC security feed for 'month' and 'year' into 'workDir', returning the +// path of the downloaded file. +func (msrc MSRCClient) GetFeed(month time.Month, year int) (string, error) { + d := time.Date(year, month, 1, 0, 0, 0, 0, time.UTC) + minD := time.Date(MSRCMinYear, time.January, 1, 0, 0, 0, 0, time.UTC) + + if d.Before(minD) { + return "", fmt.Errorf("min allowed date is %s", minD) + } + + if d.After(time.Now().UTC()) { + return "", errors.New("date can't be in the future") + } + + dst := filepath.Join(msrc.workDir, fmt.Sprintf("%s.xml", feedName(d))) + u, err := msrc.getURL(d) + if err != nil { + return "", err + } + + if err := download.Download(msrc.client, u, dst); err != nil { + return "", err + } + + return dst, nil +} diff --git a/server/vulnerabilities/msrc/io/msrc_test.go b/server/vulnerabilities/msrc/io/msrc_test.go new file mode 100644 index 0000000000..a1f8851af4 --- /dev/null +++ b/server/vulnerabilities/msrc/io/msrc_test.go @@ -0,0 +1,61 @@ +package io + +import ( + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestMSRCClient(t *testing.T) { + t.Run("#feedName", func(t *testing.T) { + date := time.Date(2010, 10, 10, 0, 0, 0, 0, time.UTC) + require.Equal(t, "2010-Oct", feedName(date)) + }) + + t.Run("#GetFeed", func(t *testing.T) { + t.Run("with invalid args", func(t *testing.T) { + sut := NewMSRCClient(nil, "", "") + now := time.Now() + + t.Run("year is below min allowed", func(t *testing.T) { + _, err := sut.GetFeed(time.January, MSRCMinYear-1) + require.Error(t, err) + }) + + t.Run("year is above current year", func(t *testing.T) { + _, err := sut.GetFeed(time.January, now.Year()+1) + require.Error(t, err) + }) + + t.Run("provided month and year is in the future", func(t *testing.T) { + _, err := sut.GetFeed((now.AddDate(0, 1, 0)).Month(), now.Year()) + require.Error(t, err) + }) + }) + + t.Run("it downloads the feed file in the provided path", func(t *testing.T) { + date := time.Date(2021, 10, 10, 0, 0, 0, 0, time.UTC) + dir := t.TempDir() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/cvrf/v2.0/document/2021-Oct" { + w.WriteHeader(http.StatusOK) + w.Write([]byte("some payload")) + } + })) + t.Cleanup(server.Close) + + sut := NewMSRCClient(server.Client(), dir, server.URL) + result, err := sut.GetFeed(date.Month(), date.Year()) + require.NoError(t, err) + + contents, err := os.ReadFile(result) + require.NoError(t, err) + require.Equal(t, []byte("some payload"), contents) + }) + }) +} diff --git a/server/vulnerabilities/msrc/io/security_bulletin_name.go b/server/vulnerabilities/msrc/io/security_bulletin_name.go index d163b24907..fc61262241 100644 --- a/server/vulnerabilities/msrc/io/security_bulletin_name.go +++ b/server/vulnerabilities/msrc/io/security_bulletin_name.go @@ -2,12 +2,13 @@ package io import ( "errors" + "fmt" "strings" "time" ) const ( - MSRCFilePrefix = "fleet_msrc_" + mSRCFilePrefix = "fleet_msrc_" fileExt = "json" dateLayout = "2006_01_02" ) @@ -31,6 +32,11 @@ func (sbn SecurityBulletinName) date() (time.Time, error) { return time.Parse(dateLayout, timeRaw) } +func FileName(productName string, date time.Time, ext string) string { + pName := strings.Replace(productName, " ", "_", -1) + return fmt.Sprintf("%s%s-%d_%02d_%02d.%s", mSRCFilePrefix, pName, date.Year(), date.Month(), date.Day(), ext) +} + func (sbn SecurityBulletinName) Before(other SecurityBulletinName) bool { a, err := sbn.date() if err != nil { @@ -46,7 +52,7 @@ func (sbn SecurityBulletinName) Before(other SecurityBulletinName) bool { } func (sbn SecurityBulletinName) ProductName() string { - pName := strings.TrimPrefix(string(sbn), MSRCFilePrefix) + pName := strings.TrimPrefix(string(sbn), mSRCFilePrefix) parts := strings.Split(pName, "-") if len(parts) != 2 { diff --git a/server/vulnerabilities/msrc/parsed/security_bulletin.go b/server/vulnerabilities/msrc/parsed/security_bulletin.go index a004903d4b..c02f0c689e 100644 --- a/server/vulnerabilities/msrc/parsed/security_bulletin.go +++ b/server/vulnerabilities/msrc/parsed/security_bulletin.go @@ -1,5 +1,13 @@ package parsed +import ( + "encoding/json" + "errors" + "os" + + "github.com/fleetdm/fleet/v4/server/ptr" +) + type SecurityBulletin struct { // The 'product' name this bulletin targets (e.g. Windows 10) ProductName string @@ -23,6 +31,64 @@ func NewSecurityBulletin(pName string) *SecurityBulletin { } } +func UnmarshalBulletin(fPath string) (*SecurityBulletin, error) { + payload, err := os.ReadFile(fPath) + if err != nil { + return nil, err + } + + bulletin := SecurityBulletin{} + err = json.Unmarshal(payload, &bulletin) + if err != nil { + return nil, err + } + return &bulletin, nil +} + +// Merge merges in-place the contents of 'other' into the current bulletin. +func (b *SecurityBulletin) Merge(other *SecurityBulletin) error { + if b.ProductName != other.ProductName { + return errors.New("bulletins are for different products") + } + + // Products + for pID, pName := range other.Products { + if _, ok := b.Products[pID]; !ok { + b.Products[pID] = pName + } + } + + // Vulnerabilities + for cve, vuln := range other.Vulnerabities { + if _, ok := b.Vulnerabities[cve]; !ok { + newVuln := NewVulnerability(vuln.PublishedEpoch) + for pID, v := range vuln.ProductIDs { + newVuln.ProductIDs[pID] = v + } + for rID, v := range vuln.RemediatedBy { + newVuln.RemediatedBy[rID] = v + } + b.Vulnerabities[cve] = newVuln + } + } + + // Vendor fixes + for kbID, r := range other.VendorFixes { + if _, ok := b.VendorFixes[kbID]; !ok { + newVF := NewVendorFix(r.FixedBuild) + for pID, v := range r.ProductIDs { + newVF.ProductIDs[pID] = v + } + if r.Supersedes != nil { + newVF.Supersedes = ptr.Int(*r.Supersedes) + } + b.VendorFixes[kbID] = newVF + } + } + + return nil +} + type Vulnerability struct { PublishedEpoch *int64 // Set of products that are susceptible to this vuln. diff --git a/server/vulnerabilities/msrc/parsed/security_bulletin_test.go b/server/vulnerabilities/msrc/parsed/security_bulletin_test.go new file mode 100644 index 0000000000..a041df751a --- /dev/null +++ b/server/vulnerabilities/msrc/parsed/security_bulletin_test.go @@ -0,0 +1,108 @@ +package parsed + +import ( + "testing" + + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/stretchr/testify/require" +) + +func TestSecurityBulletin(t *testing.T) { + t.Run("#Merge", func(t *testing.T) { + t.Run("fails if product names don't match", func(t *testing.T) { + a := NewSecurityBulletin("Windows 10") + b := NewSecurityBulletin("Windows 11") + require.Error(t, a.Merge(b)) + }) + + t.Run("with empty bulletins", func(t *testing.T) { + a := NewSecurityBulletin("Windows 10") + b := NewSecurityBulletin("Windows 10") + require.NoError(t, a.Merge(b)) + }) + + t.Run(".Products", func(t *testing.T) { + a := NewSecurityBulletin("Windows 10") + a.Products["123"] = "Windows 10 A" + a.Products["456"] = "Windows 10 B" + + b := NewSecurityBulletin("Windows 10") + a.Products["780"] = "Windows 10 C" + a.Products["980"] = "Windows 10 D" + + a.Merge(b) + + require.Equal(t, a.Products["123"], "Windows 10 A") + require.Equal(t, a.Products["456"], "Windows 10 B") + require.Equal(t, a.Products["780"], "Windows 10 C") + require.Equal(t, a.Products["980"], "Windows 10 D") + }) + + t.Run(".Vulnerabities", func(t *testing.T) { + cve1 := NewVulnerability(ptr.Int64(123)) + cve1.ProductIDs = map[string]bool{"111": true, "222": true} + cve1.RemediatedBy = map[int]bool{1: true} + + cve2 := NewVulnerability(ptr.Int64(456)) + cve2.ProductIDs = map[string]bool{"333": true, "444": true} + cve2.RemediatedBy = map[int]bool{2: true} + + cve3 := NewVulnerability(ptr.Int64(555)) + cve3.ProductIDs = map[string]bool{"aaa": true, "bbb": true} + cve3.RemediatedBy = map[int]bool{3: true} + + cve4 := NewVulnerability(ptr.Int64(777)) + cve4.ProductIDs = map[string]bool{"ccc": true, "ddd": true} + cve3.RemediatedBy = map[int]bool{4: true} + + a := NewSecurityBulletin("Windows 10") + a.Vulnerabities["cve-1"] = cve1 + a.Vulnerabities["cve-2"] = cve2 + + b := NewSecurityBulletin("Windows 10") + b.Vulnerabities["cve-3"] = cve3 + b.Vulnerabities["cve-4"] = cve4 + + a.Merge(b) + + require.Equal(t, *a.Vulnerabities["cve-1"].PublishedEpoch, int64(123)) + require.Equal(t, *a.Vulnerabities["cve-2"].PublishedEpoch, int64(456)) + require.Equal(t, *a.Vulnerabities["cve-3"].PublishedEpoch, int64(555)) + require.Equal(t, *a.Vulnerabities["cve-4"].PublishedEpoch, int64(777)) + + require.Equal(t, a.Vulnerabities["cve-1"].ProductIDs, cve1.ProductIDs) + require.Equal(t, a.Vulnerabities["cve-2"].ProductIDs, cve2.ProductIDs) + require.Equal(t, a.Vulnerabities["cve-3"].ProductIDs, cve3.ProductIDs) + require.Equal(t, a.Vulnerabities["cve-4"].ProductIDs, cve4.ProductIDs) + + require.Equal(t, a.Vulnerabities["cve-1"].RemediatedBy, cve1.RemediatedBy) + require.Equal(t, a.Vulnerabities["cve-2"].RemediatedBy, cve2.RemediatedBy) + require.Equal(t, a.Vulnerabities["cve-3"].RemediatedBy, cve3.RemediatedBy) + require.Equal(t, a.Vulnerabities["cve-4"].RemediatedBy, cve4.RemediatedBy) + }) + + t.Run(".VendorFixes", func(t *testing.T) { + vf1 := NewVendorFix("1") + vf1.ProductIDs = map[string]bool{"111": true, "222": true} + vf1.Supersedes = ptr.Int(1) + + vf2 := NewVendorFix("2") + vf2.ProductIDs = map[string]bool{"333": true, "444": true} + vf2.Supersedes = ptr.Int(2) + + a := NewSecurityBulletin("Windows 10") + a.VendorFixes[1] = vf1 + + b := NewSecurityBulletin("Windows 10") + b.VendorFixes[2] = vf2 + + a.Merge(b) + + require.Equal(t, *a.VendorFixes[1].Supersedes, int(1)) + require.Equal(t, *a.VendorFixes[2].Supersedes, int(2)) + + require.Equal(t, a.VendorFixes[1].ProductIDs, vf1.ProductIDs) + require.Equal(t, a.VendorFixes[2].ProductIDs, vf2.ProductIDs) + }) + }) +} diff --git a/server/vulnerabilities/msrc/parser.go b/server/vulnerabilities/msrc/parser.go index 7fac3d5550..6a208b6d6c 100644 --- a/server/vulnerabilities/msrc/parser.go +++ b/server/vulnerabilities/msrc/parser.go @@ -11,8 +11,8 @@ import ( msrcxml "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/xml" ) -func parseFeed(feedFilePath string) (map[string]*parsed.SecurityBulletin, error) { - r, err := os.Open(feedFilePath) +func ParseFeed(fPath string) (map[string]*parsed.SecurityBulletin, error) { + r, err := os.Open(fPath) if err != nil { return nil, fmt.Errorf("msrc parser: %w", err) } @@ -32,12 +32,17 @@ func parseFeed(feedFilePath string) (map[string]*parsed.SecurityBulletin, error) } func mapToSecurityBulletins(rXML *msrcxml.FeedResult) (map[string]*parsed.SecurityBulletin, error) { - // We will have one bulletin for each product. + // We will have one bulletin for each product name. bulletins := make(map[string]*parsed.SecurityBulletin) pIDToPName := make(map[string]string, len(rXML.WinProducts)) for pID, p := range rXML.WinProducts { name := parsed.NewProduct(p.FullName).Name() + // If the name could not be determined means that we have an un-supported Windows product + if name == "" { + continue + } + if bulletins[name] == nil { bulletins[name] = parsed.NewSecurityBulletin(name) } diff --git a/server/vulnerabilities/msrc/parser_test.go b/server/vulnerabilities/msrc/parser_test.go index 1271ca3111..94fc6ce68d 100644 --- a/server/vulnerabilities/msrc/parser_test.go +++ b/server/vulnerabilities/msrc/parser_test.go @@ -8,8 +8,8 @@ import ( "testing" "github.com/fleetdm/fleet/v4/server/ptr" - msrc_parsed "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/parsed" - msrc_xml "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/xml" + "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/parsed" + msrcxml "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/xml" "github.com/stretchr/testify/require" ) @@ -711,7 +711,7 @@ func TestParser(t *testing.T) { } // A random vulnerability ("CVE-2022-29137") - expectedVulns := map[string]map[string]msrc_parsed.Vulnerability{ + expectedVulns := map[string]map[string]parsed.Vulnerability{ "Windows 10": { "CVE-2022-29137": { PublishedEpoch: ptr.Int64(1652169600), @@ -909,7 +909,7 @@ func TestParser(t *testing.T) { } // A random vulnerability ("CVE-2022-29137") - expectedVendorFixes := map[string]map[int]msrc_parsed.VendorFix{ + expectedVendorFixes := map[string]map[int]parsed.VendorFix{ "Windows 10": { 5013941: { FixedBuild: "10.0.17763.2928", @@ -1137,9 +1137,9 @@ func TestParser(t *testing.T) { }, } - t.Run("parseFeed", func(t *testing.T) { + t.Run("ParseFeed", func(t *testing.T) { t.Run("errors out if file does not exists", func(t *testing.T) { - _, err := parseFeed("asdcv") + _, err := ParseFeed("asdcv") require.Error(t, err) }) }) @@ -1212,17 +1212,17 @@ func TestParser(t *testing.T) { t.Run("parseXML", func(t *testing.T) { t.Run("only windows products are included", func(t *testing.T) { - var expected []msrc_xml.Product + var expected []msrcxml.Product for _, grp := range expectedProducts { for pID, pFn := range grp { expected = append( expected, - msrc_xml.Product{ProductID: pID, FullName: pFn}, + msrcxml.Product{ProductID: pID, FullName: pFn}, ) } } - var actual []msrc_xml.Product + var actual []msrcxml.Product for _, v := range xmlResult.WinProducts { actual = append(actual, v) } @@ -1265,7 +1265,7 @@ func TestParser(t *testing.T) { t.Run("the remediations are parsed correctly", func(t *testing.T) { // Check the remediations of a random CVE (CVE-2022-29126) - expectedRemediations := []msrc_xml.VulnerabilityRemediation{ + expectedRemediations := []msrcxml.VulnerabilityRemediation{ { Type: "Vendor Fix", FixedBuild: "10.0.17763.2928", diff --git a/server/vulnerabilities/msrc/sync.go b/server/vulnerabilities/msrc/sync.go index dedff85a4f..6df3ad6556 100644 --- a/server/vulnerabilities/msrc/sync.go +++ b/server/vulnerabilities/msrc/sync.go @@ -7,6 +7,7 @@ import ( "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/io" "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/parsed" + "github.com/google/go-github/v37/github" ) // bulletinsDelta returns what bulletins should be download from GH and what bulletins should be removed @@ -60,8 +61,9 @@ func bulletinsDelta( // bulletin published in Github. // If 'os' is nil, then all security bulletins will be synched. func Sync(client *http.Client, dstDir string, os []fleet.OperatingSystem) error { - gh := io.NewMSRCGithubClient(client, dstDir) - fs := io.NewMSRCFSClient(dstDir) + rep := github.NewClient(client).Repositories + gh := io.NewGitHubClient(client, rep, dstDir) + fs := io.NewFSClient(dstDir) if err := sync(os, fs, gh); err != nil { return fmt.Errorf("msrc sync: %w", err) @@ -72,8 +74,8 @@ func Sync(client *http.Client, dstDir string, os []fleet.OperatingSystem) error func sync( os []fleet.OperatingSystem, - fsClient io.MSRCFSAPI, - ghClient io.MSRCGithubAPI, + fsClient io.FSAPI, + ghClient io.GitHubAPI, ) error { remoteURLs, err := ghClient.Bulletins() if err != nil { @@ -92,7 +94,7 @@ func sync( toDownload, toDelete := bulletinsDelta(os, local, remote) for _, b := range toDownload { - if err := ghClient.Download(b, remoteURLs[b]); err != nil { + if _, err := ghClient.Download(remoteURLs[b]); err != nil { return err } } diff --git a/server/vulnerabilities/msrc/sync_test.go b/server/vulnerabilities/msrc/sync_test.go index 6bf60244d5..394737e2e4 100644 --- a/server/vulnerabilities/msrc/sync_test.go +++ b/server/vulnerabilities/msrc/sync_test.go @@ -25,9 +25,9 @@ func (gh ghMock) Bulletins() (map[io.SecurityBulletinName]string, error) { return gh.testData.remoteList, gh.testData.remoteListError } -func (gh ghMock) Download(b io.SecurityBulletinName, url string) error { +func (gh ghMock) Download(url string) (string, error) { gh.testData.remoteDownloaded = append(gh.testData.remoteDownloaded, url) - return gh.testData.remoteDownloadError + return "", gh.testData.remoteDownloadError } type fsMock struct{ testData *testData }