Feature 7077: Add MSRC feed parser (#7424)

Added parser for MSRC
This commit is contained in:
Juan Fernandez
2022-08-30 16:39:50 -04:00
committed by GitHub
parent cfe338dac7
commit 2699c22143
20 changed files with 2961 additions and 0 deletions
+2
View File
@@ -0,0 +1,2 @@
- Added the MSRC feed parser that we will be using for generating the MSRC artifacts.
- Added sync logic for keeping the local MSRC artifacts up to date.
+4
View File
@@ -52,3 +52,7 @@ func Float64Ptr(x float64) **float64 {
p := Float64(x)
return &p
}
func Int64(x int64) *int64 {
return &x
}
+47
View File
@@ -0,0 +1,47 @@
package io
import (
"os"
"path/filepath"
"strings"
)
type MSRCFSAPI interface {
Bulletins() ([]SecurityBulletinName, error)
Delete(SecurityBulletinName) error
}
type MSRCFSClient struct {
dir string
}
func NewMSRCFSClient(dir string) MSRCFSClient {
return MSRCFSClient{
dir: dir,
}
}
// Delete deletes the provided security bulletin name from 'dir'.
func (fs MSRCFSClient) Delete(b SecurityBulletinName) error {
path := filepath.Join(fs.dir, string(b))
return os.Remove(path)
}
// Bulletins walks 'dir' returning all security bulletin names.
func (fs MSRCFSClient) Bulletins() ([]SecurityBulletinName, error) {
var result []SecurityBulletinName
err := filepath.WalkDir(fs.dir, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
filePath := filepath.Base(path)
if strings.HasPrefix(filePath, MSRCFilePrefix) {
result = append(result, NewSecurityBulletinName(filePath))
}
return nil
})
return result, err
}
+47
View File
@@ -0,0 +1,47 @@
package io
import (
"fmt"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
)
func TestMSRCFSClient(t *testing.T) {
t.Run("#Bulletins", func(t *testing.T) {
t.Run("directory does not exists", func(t *testing.T) {
sut := NewMSRCFSClient("asdf")
_, err := sut.Bulletins()
require.Error(t, err)
})
t.Run("returns a list of file matching the MSRC file prefix", func(t *testing.T) {
path := t.TempDir()
sut := NewMSRCFSClient(path)
file1 := filepath.Join(path, "my_lyrics.json")
bulletin1 := filepath.Join(path, fmt.Sprintf("%sWindows_10-2022_10_10.json", MSRCFilePrefix))
bulletin2 := filepath.Join(path, fmt.Sprintf("%sWindows_11-2022_10_10.json", MSRCFilePrefix))
f1, err := os.Create(bulletin1)
require.NoError(t, err)
f1.Close()
f2, err := os.Create(bulletin2)
require.NoError(t, err)
f2.Close()
f3, err := os.Create(file1)
require.NoError(t, err)
f3.Close()
r, err := sut.Bulletins()
require.NoError(t, err)
require.NotContains(t, r, NewSecurityBulletinName(filepath.Base(file1)))
require.Contains(t, r, NewSecurityBulletinName(filepath.Base(bulletin1)))
require.Contains(t, r, NewSecurityBulletinName(filepath.Base(bulletin2)))
})
})
}
+70
View File
@@ -0,0 +1,70 @@
package io
import (
"context"
"fmt"
"net/http"
"net/url"
"path/filepath"
"strings"
"time"
"github.com/fleetdm/fleet/v4/pkg/download"
"github.com/google/go-github/v37/github"
)
type MSRCGithubAPI interface {
Download(SecurityBulletinName, string) error
Bulletins() (map[SecurityBulletinName]string, error)
}
type MSRCGithubClient struct {
client *http.Client
dstDir string
}
func NewMSRCGithubClient(client *http.Client, dir string) MSRCGithubClient {
return MSRCGithubClient{client: client, dstDir: dir}
}
// Downloads the security bulletin to 'dir'.
func (gh MSRCGithubClient) Download(b SecurityBulletinName, urlStr string) error {
u, err := url.Parse(urlStr)
if err != nil {
return err
}
path := filepath.Join(gh.dstDir, string(b))
return download.DownloadAndExtract(gh.client, u, path)
}
// Bulletins returns a map of 'name' => 'download URL' of the parsed security bulletins stored as assets on Github.
func (gh MSRCGithubClient) Bulletins() (map[SecurityBulletinName]string, error) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
releases, r, err := github.NewClient(gh.client).Repositories.ListReleases(
ctx,
"fleetdm",
"nvd",
&github.ListOptions{Page: 0, PerPage: 10},
)
if err != nil {
return nil, err
}
if r.StatusCode != http.StatusOK {
return nil, fmt.Errorf("github http status error: %d", r.StatusCode)
}
results := make(map[SecurityBulletinName]string)
// TODO (juan): Since the nvd repo includes both NVD and MSRC assets, we will need to do some
// filtering logic here. To be done in https://github.com/fleetdm/fleet/issues/7394.
for _, e := range releases[0].Assets {
name := e.GetName()
if strings.HasPrefix(name, MSRCFilePrefix) {
results[NewSecurityBulletinName(name)] = e.GetBrowserDownloadURL()
}
}
return results, nil
}
@@ -0,0 +1,57 @@
package io
import (
"errors"
"strings"
"time"
)
const (
MSRCFilePrefix = "fleet_msrc_"
fileExt = "json"
dateLayout = "2006_01_02"
)
// Bulletins are published as assets to GH and copies are downloaded to the local FS. The file name
// of those assets contain some useful information like the 'product name' and the date the asset was modified. This type
// provides an abstration around the asset 'file name' to allow us to easy extract/compare the encoded info.
type SecurityBulletinName string
func NewSecurityBulletinName(str string) SecurityBulletinName {
return SecurityBulletinName(str)
}
func (sbn SecurityBulletinName) date() (time.Time, error) {
parts := strings.Split(string(sbn), "-")
if len(parts) != 2 {
return time.Now(), errors.New("invalid security bulletin name")
}
timeRaw := strings.TrimSuffix(parts[1], "."+fileExt)
return time.Parse(dateLayout, timeRaw)
}
func (sbn SecurityBulletinName) Before(other SecurityBulletinName) bool {
a, err := sbn.date()
if err != nil {
return false
}
b, err := other.date()
if err != nil {
return false
}
return a.Before(b)
}
func (sbn SecurityBulletinName) ProductName() string {
pName := strings.TrimPrefix(string(sbn), MSRCFilePrefix)
parts := strings.Split(pName, "-")
if len(parts) != 2 {
return ""
}
return strings.Replace(parts[0], "_", " ", -1)
}
@@ -0,0 +1,34 @@
package io
import (
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestSecurityBulletinName(t *testing.T) {
t.Run("#date", func(t *testing.T) {
sut := NewSecurityBulletinName("Windows_10-2022_09_10.json")
result, err := sut.date()
require.NoError(t, err)
require.Equal(t, 2022, result.Year())
require.Equal(t, time.Month(9), result.Month())
require.Equal(t, 10, result.Day())
})
t.Run("#ProductName", func(t *testing.T) {
a := NewSecurityBulletinName("Windows_10-2022_09_10.json")
require.Equal(t, "Windows 10", a.ProductName())
})
t.Run("#Before", func(t *testing.T) {
a := NewSecurityBulletinName("Windows_10-2022_09_10.json")
b := NewSecurityBulletinName("Windows_10-2022_10_10.json")
c := NewSecurityBulletinName("Windows_10-2022_10_10.json")
require.True(t, a.Before(b))
require.False(t, b.Before(a))
require.False(t, b.Before(c))
require.False(t, c.Before(b))
})
}
@@ -0,0 +1,77 @@
package parsed
import "strings"
// Product abstracts a MS full product name.
// A full product name includes the name of the product plus its arch
// (if any) and its version (if any).
type Product string
func NewProduct(fullName string) Product {
return Product(fullName)
}
// Arch returns the archicture for the current Microsoft product, if none can
// be found then "all" is returned. Returned values are meant to match the values returned from
// `SELECT arch FROM os_version` in OSQuery.
// eg:
// "Windows 10 Version 1803 for 32-bit Systems" => "32-bit"
func (p Product) Arch() string {
val := string(p)
switch {
case strings.Index(val, "32-bit") != -1:
return "32-bit"
case strings.Index(val, "x64") != -1:
return "64-bit"
case strings.Index(val, "ARM64") != -1:
return "arm64"
case strings.Index(val, "Itanium-Based") != -1:
return "itanium"
default:
return "all"
}
}
// Name returns the name for the current Microsoft product, if none can
// be found then "" is returned.
// eg:
// "Windows 10 Version 1803 for 32-bit Systems" => "Windows 10"
// "Windows Server 2008 R2 for Itanium-Based Systems Service Pack 1" => "Windows Server 2008 R2"
func (p Product) Name() string {
val := string(p)
switch {
// Desktop versions
case strings.Index(val, "Windows 7") != -1:
return "Windows 7"
case strings.Index(val, "Windows 8.1") != -1:
return "Windows 8.1"
case strings.Index(val, "Windows RT 8.1") != -1:
return "Windows RT 8.1"
case strings.Index(val, "Windows 10") != -1:
return "Windows 10"
case strings.Index(val, "Windows 11") != -1:
return "Windows 11"
// Server versions
case strings.Index(val, "Windows Server 2008 R2") != -1:
return "Windows Server 2008 R2"
case strings.Index(val, "Windows Server 2012 R2") != -1:
return "Windows Server 2012 R2"
case strings.Index(val, "Windows Server 2008") != -1:
return "Windows Server 2008"
case strings.Index(val, "Windows Server 2012") != -1:
return "Windows Server 2012"
case strings.Index(val, "Windows Server 2016") != -1:
return "Windows Server 2016"
case strings.Index(val, "Windows Server 2019") != -1:
return "Windows Server 2019"
case strings.Index(val, "Windows Server 2022") != -1:
return "Windows Server 2022"
case strings.Index(val, "Windows Server,") != -1:
return "Windows Server"
default:
return ""
}
}
@@ -0,0 +1,375 @@
package parsed
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestFullProductName(t *testing.T) {
testCases := []struct {
fullName string
arch string
prodName string
}{
{
fullName: "Windows 10 Version 1809 for 32-bit Systems",
arch: "32-bit",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 1809 for x64-based Systems",
arch: "64-bit",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 1809 for ARM64-based Systems",
arch: "arm64",
prodName: "Windows 10",
},
{
fullName: "Windows Server 2019",
arch: "all",
prodName: "Windows Server 2019",
},
{
fullName: "Windows Server 2019 (Server Core installation)",
arch: "all",
prodName: "Windows Server 2019",
},
{
fullName: "Windows 10 Version 1909 for 32-bit Systems",
arch: "32-bit",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 1909 for x64-based Systems",
arch: "64-bit",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 1909 for ARM64-based Systems",
arch: "arm64",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 21H1 for x64-based Systems",
arch: "64-bit",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 21H1 for ARM64-based Systems",
arch: "arm64",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 21H1 for 32-bit Systems",
arch: "32-bit",
prodName: "Windows 10",
},
{
fullName: "Windows Server 2022",
arch: "all",
prodName: "Windows Server 2022",
},
{
fullName: "Windows Server 2022 (Server Core installation)",
arch: "all",
prodName: "Windows Server 2022",
},
{
fullName: "Windows 10 Version 20H2 for x64-based Systems",
arch: "64-bit",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 20H2 for 32-bit Systems",
arch: "32-bit",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 20H2 for ARM64-based Systems",
arch: "arm64",
prodName: "Windows 10",
},
{
fullName: "Windows Server, version 20H2 (Server Core Installation)",
arch: "all",
prodName: "Windows Server",
},
{
fullName: "Windows 11 for x64-based Systems",
arch: "64-bit",
prodName: "Windows 11",
},
{
fullName: "Windows 11 for ARM64-based Systems",
arch: "arm64",
prodName: "Windows 11",
},
{
fullName: "Windows 10 Version 21H2 for 32-bit Systems",
arch: "32-bit",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 21H2 for ARM64-based Systems",
arch: "arm64",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 21H2 for x64-based Systems",
arch: "64-bit",
prodName: "Windows 10",
},
{
fullName: "Windows 10 for 32-bit Systems",
arch: "32-bit",
prodName: "Windows 10",
},
{
fullName: "Windows 10 for x64-based Systems",
arch: "64-bit",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 1607 for 32-bit Systems",
arch: "32-bit",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 1607 for x64-based Systems",
arch: "64-bit",
prodName: "Windows 10",
},
{
fullName: "Windows Server 2016",
arch: "all",
prodName: "Windows Server 2016",
},
{
fullName: "Windows Server 2016 (Server Core installation)",
arch: "all",
prodName: "Windows Server 2016",
},
{
fullName: "Windows 8.1 for 32-bit systems",
arch: "32-bit",
prodName: "Windows 8.1",
},
{
fullName: "Windows 8.1 for x64-based systems",
arch: "64-bit",
prodName: "Windows 8.1",
},
{
fullName: "Windows RT 8.1",
arch: "all",
prodName: "Windows RT 8.1",
},
{
fullName: "Windows Server 2012",
arch: "all",
prodName: "Windows Server 2012",
},
{
fullName: "Windows Server 2012 (Server Core installation)",
arch: "all",
prodName: "Windows Server 2012",
},
{
fullName: "Windows Server 2012 R2",
arch: "all",
prodName: "Windows Server 2012 R2",
},
{
fullName: "Windows Server 2012 R2 (Server Core installation)",
arch: "all",
prodName: "Windows Server 2012 R2",
},
{
fullName: "Windows 7 for 32-bit Systems Service Pack 1",
arch: "32-bit",
prodName: "Windows 7",
},
{
fullName: "Windows 7 for x64-based Systems Service Pack 1",
arch: "64-bit",
prodName: "Windows 7",
},
{
fullName: "Windows Server 2008 for 32-bit Systems Service Pack 2",
arch: "32-bit",
prodName: "Windows Server 2008",
},
{
fullName: "Windows Server 2008 for 32-bit Systems Service Pack 2 (Server Core installation)",
arch: "32-bit",
prodName: "Windows Server 2008",
},
{
fullName: "Windows Server 2008 for x64-based Systems Service Pack 2",
arch: "64-bit",
prodName: "Windows Server 2008",
},
{
fullName: "Windows Server 2008 for x64-based Systems Service Pack 2 (Server Core installation)",
arch: "64-bit",
prodName: "Windows Server 2008",
},
{
fullName: "Windows Server 2008 R2 for x64-based Systems Service Pack 1",
arch: "64-bit",
prodName: "Windows Server 2008 R2",
},
{
fullName: "Windows Server 2008 R2 for x64-based Systems Service Pack 1 (Server Core installation)",
arch: "64-bit",
prodName: "Windows Server 2008 R2",
},
{
fullName: "Windows 10 Version 1803 for x64-based Systems",
arch: "64-bit",
prodName: "Windows 10",
},
{
fullName: "Windows Server, version 1803 (Server Core Installation)",
arch: "all",
prodName: "Windows Server",
},
{
fullName: "Windows 10 Version 1809 for x64-based Systems",
arch: "64-bit",
prodName: "Windows 10",
},
{
fullName: "Windows Server 2019",
arch: "all",
prodName: "Windows Server 2019",
},
{
fullName: "Windows Server 2019 (Server Core installation)",
arch: "all",
prodName: "Windows Server 2019",
},
{
fullName: "Windows 10 Version 1709 for x64-based Systems",
arch: "64-bit",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 1903 for x64-based Systems",
arch: "64-bit",
prodName: "Windows 10",
},
{
fullName: "Windows Server, version 1903 (Server Core installation)",
arch: "all",
prodName: "Windows Server",
},
{
fullName: "Windows 10 for x64-based Systems",
arch: "64-bit",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 1607 for x64-based Systems",
arch: "64-bit",
prodName: "Windows 10",
},
{
fullName: "Windows Server 2016",
arch: "all",
prodName: "Windows Server 2016",
},
{
fullName: "Windows Server 2016 (Server Core installation)",
arch: "all",
prodName: "Windows Server 2016",
},
{
fullName: "Windows 8.1 for x64-based systems",
arch: "64-bit",
prodName: "Windows 8.1",
},
{
fullName: "Windows Server 2012",
arch: "all",
prodName: "Windows Server 2012",
},
{
fullName: "Windows Server 2012 (Server Core installation)",
arch: "all",
prodName: "Windows Server 2012",
},
{
fullName: "Windows Server 2012 R2",
arch: "all",
prodName: "Windows Server 2012 R2",
},
{
fullName: "Windows Server 2012 R2 (Server Core installation)",
arch: "all",
prodName: "Windows Server 2012 R2",
},
{
fullName: "Windows 10 Version 1909 for x64-based Systems",
arch: "64-bit",
prodName: "Windows 10",
},
{
fullName: "Windows Server, version 1909 (Server Core installation)",
arch: "all",
prodName: "Windows Server",
},
{
fullName: "Windows 10 Version 1803 for 32-bit Systems",
arch: "32-bit",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 1803 for ARM64-based Systems",
arch: "arm64",
prodName: "Windows 10",
},
{
fullName: "Windows 10 Version 1809 for 32-bit Systems",
arch: "32-bit",
prodName: "Windows 10",
},
{
fullName: "None Available",
arch: "all",
prodName: "",
},
{
fullName: "Windows Server 2008 for 32-bit Systems Service Pack 2 (Server Core installation)",
arch: "32-bit",
prodName: "Windows Server 2008",
},
{
fullName: "Windows Server 2008 for Itanium-Based Systems Service Pack 2",
arch: "itanium",
prodName: "Windows Server 2008",
},
{
fullName: "Windows Server 2008 R2 for Itanium-Based Systems Service Pack 1",
arch: "itanium",
prodName: "Windows Server 2008 R2",
},
}
t.Run("#ArchFromProdName", func(t *testing.T) {
for _, tCase := range testCases {
sut := NewProduct(tCase.fullName)
require.Equal(t, tCase.arch, sut.Arch(), tCase)
}
})
t.Run("#NameFromFullProdName", func(t *testing.T) {
for _, tCase := range testCases {
sut := NewProduct(tCase.fullName)
require.Equal(t, tCase.prodName, sut.Name(), tCase)
}
})
}
@@ -0,0 +1,55 @@
package parsed
type SecurityBulletin struct {
// The 'product' name this bulletin targets (e.g. Windows 10)
ProductName string
// All products contained in this bulletin (Product ID => Product full name).
// We can have many different 'products' under a single name, for example, for 'Windows 10':
// - Windows 10 Version 1809 for 32-bit Systems
// - Windows 10 Version 1909 for x64-based Systems
Products map[string]string
// All vulnerabilities contained in this bulletin, by CVE
Vulnerabities map[string]Vulnerability
// All vendor fixes for remediating the vulnerabilities contained in this bulletin, by KBID
VendorFixes map[int]VendorFix
}
func NewSecurityBulletin(pName string) *SecurityBulletin {
return &SecurityBulletin{
ProductName: pName,
Products: make(map[string]string),
Vulnerabities: make(map[string]Vulnerability),
VendorFixes: make(map[int]VendorFix),
}
}
type Vulnerability struct {
PublishedEpoch *int64
// Set of products that are susceptible to this vuln.
ProductIDs map[string]bool
// Set of Vendor fixes that remediate this vuln.
RemediatedBy map[int]bool
}
func NewVulnerability(publishedDateEpoch *int64) Vulnerability {
return Vulnerability{
PublishedEpoch: publishedDateEpoch,
ProductIDs: make(map[string]bool),
RemediatedBy: make(map[int]bool),
}
}
type VendorFix struct {
// TODO (juan): Do we need this?
FixedBuild string
ProductIDs map[string]bool
// A Reference to what vendor fix this particular vendor fix 'replaces'.
Supersedes *int `json:",omitempty"`
}
func NewVendorFix(fixedBuild string) VendorFix {
return VendorFix{
FixedBuild: fixedBuild,
ProductIDs: make(map[string]bool),
}
}
+153
View File
@@ -0,0 +1,153 @@
package msrc
import (
"encoding/xml"
"fmt"
"io"
"os"
"strconv"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/parsed"
msrcxml "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/xml"
)
func parseFeed(feedFilePath string) (map[string]*parsed.SecurityBulletin, error) {
r, err := os.Open(feedFilePath)
if err != nil {
return nil, fmt.Errorf("msrc parser: %w", err)
}
defer r.Close()
feedResultXML, err := parseXML(r)
if err != nil {
return nil, fmt.Errorf("msrc parser: %w", err)
}
bulletins, err := mapToSecurityBulletins(feedResultXML)
if err != nil {
return nil, fmt.Errorf("msrc parser: %w", err)
}
return bulletins, nil
}
func mapToSecurityBulletins(rXML *msrcxml.FeedResult) (map[string]*parsed.SecurityBulletin, error) {
// We will have one bulletin for each product.
bulletins := make(map[string]*parsed.SecurityBulletin)
pIDToPName := make(map[string]string, len(rXML.WinProducts))
for pID, p := range rXML.WinProducts {
name := parsed.NewProduct(p.FullName).Name()
if bulletins[name] == nil {
bulletins[name] = parsed.NewSecurityBulletin(name)
}
bulletins[name].Products[pID] = p.FullName
pIDToPName[pID] = name
}
for _, v := range rXML.WinVulnerabities {
for _, rem := range v.Remediations {
// We will only be able to detect vulns for which they are vendor fixes.
if !rem.IsVendorFix() {
continue
}
// We assume that rem.Description will contain the ID portion of a KBID, which should
// be always a numeric value.
remediatedKBID, err := strconv.Atoi(rem.Description)
if err != nil {
return nil, fmt.Errorf("invalid remediation KBID %q for %s", rem.Description, v.CVE)
}
// rem.Supercedence should have the ID portion of a KBID which the current vendor fix replaces.
var supersedes *int
if rem.Supercedence != "" {
r, err := strconv.Atoi(rem.Supercedence)
if err != nil {
return nil, fmt.Errorf("invalid supercedence KBID %q for %s", rem.Supercedence, v.CVE)
}
supersedes = &r
}
for _, pID := range rem.ProductIDs {
// Get the bulletin for the current product ID, skip further processing if is a
// non-windows product.
b, ok := bulletins[pIDToPName[pID]]
if !ok {
continue
}
// Check if the vulnerability referenced by this remediation exists, if not
// initialize it.
var vuln parsed.Vulnerability
if vuln, ok = b.Vulnerabities[v.CVE]; !ok {
vuln = parsed.NewVulnerability(v.PublishedDateEpoch())
}
vuln.ProductIDs[pID] = true
vuln.RemediatedBy[remediatedKBID] = true
// Check if the vendor fix referenced by this remediation exists, if not
// initialize it.
var vFix parsed.VendorFix
if vFix, ok = b.VendorFixes[remediatedKBID]; !ok {
vFix = parsed.NewVendorFix(rem.FixedBuild)
}
vFix.Supersedes = supersedes
vFix.ProductIDs[pID] = true
// Update the bulletin
b.Vulnerabities[v.CVE] = vuln
b.VendorFixes[remediatedKBID] = vFix
}
}
}
return bulletins, nil
}
func parseXML(reader io.Reader) (*msrcxml.FeedResult, error) {
r := &msrcxml.FeedResult{
WinProducts: map[string]msrcxml.Product{},
}
d := xml.NewDecoder(reader)
for {
t, err := d.Token()
if err != nil {
if err == io.EOF {
return r, nil
}
return nil, fmt.Errorf("decoding token: %v", err)
}
switch t := t.(type) {
case xml.StartElement:
if t.Name.Local == "Branch" {
branch := msrcxml.ProductBranch{}
if err = d.DecodeElement(&branch, &t); err != nil {
return nil, err
}
for _, p := range branch.WinProducts() {
r.WinProducts[p.ProductID] = p
}
}
if t.Name.Local == "Vulnerability" {
vuln := msrcxml.Vulnerability{}
if err = d.DecodeElement(&vuln, &t); err != nil {
return nil, err
}
for pID := range r.WinProducts {
// We only care about vulnerabilities that have a vendor fix targeting a Windows
// product.
if vuln.IncludesVendorFix(pID) {
r.WinVulnerabities = append(r.WinVulnerabities, vuln)
break
}
}
}
}
}
}
File diff suppressed because it is too large Load Diff
+106
View File
@@ -0,0 +1,106 @@
package msrc
import (
"fmt"
"net/http"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/io"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/parsed"
)
// bulletinsDelta returns what bulletins should be download from GH and what bulletins should be removed
// from the local file system based what OS are installed, what local bulletins we have and what
// remote bulletins exist.
func bulletinsDelta(
os []fleet.OperatingSystem,
local []io.SecurityBulletinName,
remote []io.SecurityBulletinName,
) (
[]io.SecurityBulletinName,
[]io.SecurityBulletinName,
) {
if len(os) == 0 {
return remote, nil
}
var matching []io.SecurityBulletinName
for _, r := range remote {
for _, o := range os {
product := parsed.NewProduct(o.Name)
if r.ProductName() == product.Name() {
matching = append(matching, r)
}
}
}
var toDownload []io.SecurityBulletinName
var toDelete []io.SecurityBulletinName
for _, m := range matching {
var found bool
for _, l := range local {
if m.ProductName() == l.ProductName() {
found = true
// out of date
if l.Before(m) {
toDownload = append(toDownload, m)
toDelete = append(toDelete, l)
}
break
}
}
if !found {
toDownload = append(toDownload, m)
}
}
return toDownload, toDelete
}
// Sync syncs the local msrc security bulletins (contained in dstDir) for one or more operating systems with the security
// bulletin published in Github.
// If 'os' is nil, then all security bulletins will be synched.
func Sync(client *http.Client, dstDir string, os []fleet.OperatingSystem) error {
gh := io.NewMSRCGithubClient(client, dstDir)
fs := io.NewMSRCFSClient(dstDir)
if err := sync(os, fs, gh); err != nil {
return fmt.Errorf("msrc sync: %w", err)
}
return nil
}
func sync(
os []fleet.OperatingSystem,
fsClient io.MSRCFSAPI,
ghClient io.MSRCGithubAPI,
) error {
remoteURLs, err := ghClient.Bulletins()
if err != nil {
return err
}
var remote []io.SecurityBulletinName
for r := range remoteURLs {
remote = append(remote, r)
}
local, err := fsClient.Bulletins()
if err != nil {
return err
}
toDownload, toDelete := bulletinsDelta(os, local, remote)
for _, b := range toDownload {
if err := ghClient.Download(b, remoteURLs[b]); err != nil {
return err
}
}
for _, d := range toDelete {
if err := fsClient.Delete(d); err != nil {
return err
}
}
return nil
}
+197
View File
@@ -0,0 +1,197 @@
package msrc
import (
"testing"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc/io"
"github.com/stretchr/testify/require"
)
type testData struct {
remoteList map[io.SecurityBulletinName]string
remoteListError error
remoteDownloaded []string
remoteDownloadError error
localList []io.SecurityBulletinName
localListError error
localDeleted []io.SecurityBulletinName
localDeleteError error
}
type ghMock struct{ testData *testData }
func (gh ghMock) Bulletins() (map[io.SecurityBulletinName]string, error) {
return gh.testData.remoteList, gh.testData.remoteListError
}
func (gh ghMock) Download(b io.SecurityBulletinName, url string) error {
gh.testData.remoteDownloaded = append(gh.testData.remoteDownloaded, url)
return gh.testData.remoteDownloadError
}
type fsMock struct{ testData *testData }
func (fs fsMock) Bulletins() ([]io.SecurityBulletinName, error) {
return fs.testData.localList, fs.testData.localListError
}
func (fs fsMock) Delete(d io.SecurityBulletinName) error {
fs.testData.localDeleted = append(fs.testData.localDeleted, d)
return fs.testData.localDeleteError
}
func TestSync(t *testing.T) {
t.Run("#sync", func(t *testing.T) {
os := []fleet.OperatingSystem{
{
Name: "Microsoft Windows 11 Enterprise",
Version: "21H2",
Arch: "64-bit",
KernelVersion: "10.0.22000.795",
},
{
Name: "Microsoft Windows 10 Pro",
Version: "10.0.19044",
Arch: "64-bit",
KernelVersion: "10.0.19044",
},
}
testData := testData{
remoteList: map[io.SecurityBulletinName]string{
io.NewSecurityBulletinName("Windows_10-2022_10_10.json"): "http://somebulletin.com",
},
localList: []io.SecurityBulletinName{"Windows_10-2022_09_10.json"},
}
err := sync(os, fsMock{testData: &testData}, ghMock{testData: &testData})
require.NoError(t, err)
require.ElementsMatch(t, testData.remoteDownloaded, []string{"http://somebulletin.com"})
require.ElementsMatch(t, testData.localDeleted, []io.SecurityBulletinName{"Windows_10-2022_09_10.json"})
})
t.Run("#bulletinsDelta", func(t *testing.T) {
t.Run("win OS provided", func(t *testing.T) {
os := []fleet.OperatingSystem{
{
Name: "Microsoft Windows 11 Enterprise",
Version: "21H2",
Arch: "64-bit",
KernelVersion: "10.0.22000.795",
},
{
Name: "Microsoft Windows 10 Pro",
Version: "10.0.19044",
Arch: "64-bit",
KernelVersion: "10.0.19044",
},
}
t.Run("without remote bulletins", func(t *testing.T) {
var remote []io.SecurityBulletinName
local := []io.SecurityBulletinName{
"Windows_10-2022_10_10.json",
}
toDownload, toDelete := bulletinsDelta(os, local, remote)
require.Empty(t, toDownload)
require.Empty(t, toDelete)
})
t.Run("with remote bulletins", func(t *testing.T) {
remote := []io.SecurityBulletinName{
"Windows_10-2022_10_10.json",
"Windows_11-2022_10_10.json",
"Windows_Server_2016-2022_10_10.json",
"Windows_8.1-2022_10_10.json",
}
t.Run("no local bulletins", func(t *testing.T) {
var local []io.SecurityBulletinName
toDownload, toDelete := bulletinsDelta(os, local, remote)
require.ElementsMatch(t, toDownload, []io.SecurityBulletinName{
"Windows_10-2022_10_10.json",
"Windows_11-2022_10_10.json",
})
require.Empty(t, toDelete)
})
t.Run("missing some local bulletin", func(t *testing.T) {
local := []io.SecurityBulletinName{
"Windows_10-2022_10_10.json",
}
toDownload, toDelete := bulletinsDelta(os, local, remote)
require.ElementsMatch(t, toDownload, []io.SecurityBulletinName{
"Windows_11-2022_10_10.json",
})
require.Empty(t, toDelete)
})
t.Run("out of date local bulletin", func(t *testing.T) {
local := []io.SecurityBulletinName{
"Windows_10-2022_09_10.json",
"Windows_11-2022_10_10.json",
}
toDownload, toDelete := bulletinsDelta(os, local, remote)
require.ElementsMatch(t, toDownload, []io.SecurityBulletinName{
"Windows_10-2022_10_10.json",
})
require.ElementsMatch(t, toDelete, []io.SecurityBulletinName{
"Windows_10-2022_09_10.json",
})
})
t.Run("up to date local bulletins", func(t *testing.T) {
local := []io.SecurityBulletinName{
"Windows_10-2022_10_10.json",
"Windows_11-2022_10_10.json",
}
toDownload, toDelete := bulletinsDelta(os, local, remote)
require.Empty(t, toDownload)
require.Empty(t, toDelete)
})
})
})
t.Run("no Win OS provided", func(t *testing.T) {
os := []fleet.OperatingSystem{
{
Name: "CentOS",
Version: "8.0.0",
Platform: "rhel",
KernelVersion: "5.10.76-linuxkit",
},
}
local := []io.SecurityBulletinName{"Windows_11-2022_10_10.json"}
remote := []io.SecurityBulletinName{"Windows_10-2022_10_10.json"}
t.Run("nothing to download, nothing to delete", func(t *testing.T) {
toDownload, toDelete := bulletinsDelta(os, local, remote)
require.Empty(t, toDownload)
require.Empty(t, toDelete)
})
})
t.Run("no OS provided", func(t *testing.T) {
var os []fleet.OperatingSystem
t.Run("no local bulletins", func(t *testing.T) {
var local []io.SecurityBulletinName
t.Run("returns all remote", func(t *testing.T) {
remote := []io.SecurityBulletinName{
"Windows_10-2022_10_10.json",
"Windows_11-2022_10_10.json",
}
toDownload, toDelete := bulletinsDelta(os, local, remote)
require.ElementsMatch(t, toDownload, remote)
require.Empty(t, toDelete)
})
})
})
})
}
@@ -0,0 +1,7 @@
package xml
// FeedResult groups together products and their vulnerabilities.
type FeedResult struct {
WinVulnerabities []Vulnerability
WinProducts map[string]Product
}
@@ -0,0 +1,48 @@
package xml
import "strings"
// XML elements related to the 'prod' namespace used to describe Microsoft products
// Describes a product three see
// http://docs.oasis-open.org/csaf/csaf-cvrf/v1.2/cs01/csaf-cvrf-v1.2-cs01.html#_Toc493508797
// for more details
type ProductBranch struct {
Type string `xml:"Type,attr"`
Name string `xml:"Name,attr"`
Branches []ProductBranch `xml:"Branch"`
Products []Product `xml:"FullProductName"`
}
// Describes a full product name
// http://docs.oasis-open.org/csaf/csaf-cvrf/v1.2/cs01/csaf-cvrf-v1.2-cs01.html#_Toc493508797
type Product struct {
ProductID string `xml:"ProductID,attr"`
FullName string `xml:",chardata"`
}
// WinProducts traverses the ProductBranchXML tree returning only 'Windows' products.
func (b *ProductBranch) WinProducts() []Product {
var r []Product
queue := []ProductBranch{*b}
for len(queue) > 0 {
next := queue[0]
// We want only products from the 'Windows' and the 'Extended Security Update (ESU)' branches
if next.Type == "Product Family" && (next.Name == "Windows" || next.Name == "ESU") {
for _, p := range next.Products {
// Even if the product branch is for 'Windows/ESU', there could be a non-OS
// product like 'Remote Desktop client for Windows Desktop' inside the branch.
if strings.HasPrefix(p.FullName, "Windows") {
r = append(r, p)
}
}
}
queue = queue[1:]
queue = append(queue, next.Branches...)
}
return r
}
@@ -0,0 +1,53 @@
package xml
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestProdXML(t *testing.T) {
t.Run("ProductBranchXML", func(t *testing.T) {
t.Run("#WindowsProducts", func(t *testing.T) {
windowsBranch := ProductBranch{
Type: "Product Family", Name: "Windows",
Products: []Product{
{ProductID: "11572", FullName: "Windows Server 2019 (Server Core installation)"},
{ProductID: "11712", FullName: "Windows 10 Version 1909 for 32-bit Systems"},
},
}
esuBranch := ProductBranch{
Type: "Product Family", Name: "ESU",
Products: []Product{
{ProductID: "10051", FullName: "Windows Server 2008 R2 for x64-based Systems Service Pack 1"},
{ProductID: "10049", FullName: "Windows Server 2008 R2 for x64-based Systems Service Pack 1 (Server Core installation)"},
},
}
devToolsBranch := ProductBranch{
Type: "Product Family", Name: "Developer Tools",
Products: []Product{
{ProductID: "11676-11927", FullName: "Microsoft .NET Framework 3.5 AND 4.8 on Windows 11 for ARM64-based Systems"},
{ProductID: "9495-10047", FullName: "Microsoft .NET Framework 3.5.1 on Windows 7 for 32-bit Systems Service Pack 1"},
{ProductID: "9495-10048", FullName: "Microsoft .NET Framework 3.5.1 on Windows 7 for x64-based Systems Service Pack 1"},
{ProductID: "9495-10051", FullName: "Microsoft .NET Framework 3.5.1 on Windows Server 2008 R2 for x64-based Systems Service Pack 1"},
},
}
rootBranch := &ProductBranch{
Type: "Vendor", Name: "Microsoft",
Branches: []ProductBranch{
windowsBranch,
esuBranch,
devToolsBranch,
},
}
winProds := rootBranch.WinProducts()
require.Subset(t, winProds, windowsBranch.Products)
require.Subset(t, winProds, esuBranch.Products)
require.NotSubset(t, winProds, devToolsBranch.Products)
})
})
}
@@ -0,0 +1,73 @@
package xml
import (
"fmt"
"strings"
"time"
)
// XML elements related to the 'vuln' namespace used to describe vulnerabilities and their remediations.
// Vulnerability see
// http://docs.oasis-open.org/csaf/csaf-cvrf/v1.2/cs01/csaf-cvrf-v1.2-cs01.html#_Toc493508834
// for more details.
type Vulnerability struct {
CVE string `xml:"CVE"`
Score float64 `xml:"CVSSScoreSets>ScoreSet>BaseScore"`
Revisions []RevisionHistory `xml:"RevisionHistory>Revision"`
Remediations []VulnerabilityRemediation `xml:"Remediations>Remediation"`
}
type RevisionHistory struct {
Date string `xml:"Date"`
Description string `xml:"Description"`
}
// VulnerabilityRemediation See http://docs.oasis-open.org/csaf/csaf-cvrf/v1.2/cs01/csaf-cvrf-v1.2-cs01.html#_Toc493508854
// for more details.
type VulnerabilityRemediation struct {
Type string `xml:"Type,attr"`
FixedBuild string `xml:"FixedBuild"`
RestartRequired string `xml:"RestartRequired"`
ProductIDs []string `xml:"ProductID"`
Description string `xml:"Description"`
URL string `xml:"URL"`
Supercedence string `xml:"Supercedence"`
}
// IncludesVendorFix returns true if the vulnerability has a vendor fix targeting the product
// identified by pID.
func (v *Vulnerability) IncludesVendorFix(pID string) bool {
for _, rem := range v.Remediations {
if rem.IsVendorFix() {
for _, vfPID := range rem.ProductIDs {
if vfPID == pID {
return true
}
}
}
}
return false
}
// PublishedDateEpoch returns the date the vuln was published (if any) as an epoch
func (v *Vulnerability) PublishedDateEpoch() *int64 {
for _, rev := range v.Revisions {
if strings.Index(rev.Description, "Information published") != -1 {
dPublished, err := time.Parse("2006-01-02T15:04:05", rev.Date)
if err != nil {
return nil
}
epoch := dPublished.Unix()
return &epoch
}
}
return nil
}
func (rem *VulnerabilityRemediation) IsVendorFix() bool {
return rem.Type == "Vendor Fix" &&
strings.HasPrefix(rem.URL, "https://catalog.update") &&
strings.HasSuffix(rem.URL, fmt.Sprintf("q=KB%s", rem.Description))
}
@@ -0,0 +1,89 @@
package xml
import (
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestVulnunerability(t *testing.T) {
t.Run("VulnerabilityXML", func(t *testing.T) {
t.Run("#PublishedDateEpoch", func(t *testing.T) {
sut := Vulnerability{
Revisions: []RevisionHistory{
{
Description: "<p>Information published.</p> ",
Date: "2022-05-10T07:00:00",
},
},
}
resultEpoch := sut.PublishedDateEpoch()
require.NotNil(t, resultEpoch)
resultDate := time.Unix(*resultEpoch, 0)
require.Equal(t, 2022, resultDate.Year())
require.Equal(t, time.May, resultDate.Month())
require.Equal(t, 10, resultDate.Day())
})
t.Run("#IncludesVendorFix", func(t *testing.T) {
t.Run("no remediations", func(t *testing.T) {
sut := Vulnerability{}
require.False(t, sut.IncludesVendorFix("1"))
})
t.Run("no vendor fixes", func(t *testing.T) {
sut := Vulnerability{
Remediations: []VulnerabilityRemediation{
{
Type: "Known Issue",
ProductIDs: []string{"11896", "11897"},
Description: "5013942",
URL: "https://support.microsoft.com/help/5013942",
},
},
}
require.False(t, sut.IncludesVendorFix("11896"))
})
t.Run("no vendor fix matches", func(t *testing.T) {
sut := Vulnerability{
Remediations: []VulnerabilityRemediation{
{
Type: "Vendor Fix",
FixedBuild: "10.0.17763.2928",
ProductIDs: []string{"11568", "11569"},
Description: "5013941",
Supercedence: "5012647",
RestartRequired: "Yes",
URL: "https://catalog.update.microsoft.com/v7/site/Search.aspx?q=KB5013941",
},
},
}
require.False(t, sut.IncludesVendorFix("123"))
})
t.Run("vendor fix matches", func(t *testing.T) {
sut := Vulnerability{
Remediations: []VulnerabilityRemediation{
{
Type: "Vendor Fix",
FixedBuild: "10.0.17763.2928",
ProductIDs: []string{"11568", "11569"},
Description: "5013941",
Supercedence: "5012647",
RestartRequired: "Yes",
URL: "https://catalog.update.microsoft.com/v7/site/Search.aspx?q=KB5013941",
},
},
}
require.True(t, sut.IncludesVendorFix("11568"))
})
})
})
}
Binary file not shown.