Trigger webhooks for recently published vulnerabilities (#3941)

2022-02-02 16:34:37 -05:00
parent b90e2e2e3d
commit 6e2ba62744
15 changed files with 523 additions and 42 deletions
@@ -11,7 +11,9 @@ import (
 	"sync"
 	"time"

+	"github.com/WatchBeam/clock"
 	"github.com/facebookincubator/nvdtools/cvefeed"
+	feednvd "github.com/facebookincubator/nvdtools/cvefeed/nvd"
 	"github.com/facebookincubator/nvdtools/providers/nvd"
 	"github.com/facebookincubator/nvdtools/wfn"
 	"github.com/fleetdm/fleet/v4/server/config"
@@ -49,60 +51,89 @@ func SyncCVEData(vulnPath string, config config.FleetConfig) error {
 	return dfs.Do(ctx)
 }

+const publishedDateFmt = "2006-01-02T15:04Z" // not quite RFC3339
+
+var (
+	rxNVDCVEArchive = regexp.MustCompile(`nvdcve.*\.gz$`)
+
+	// max age to be considered a recent vulnerability (relative to NVD's published date)
+	// (a var to be able to change in tests)
+	recentVulnMaxAge = 2 * 24 * time.Hour
+
+	// this allows mocking the time package for tests, by default it is equivalent
+	// to the time functions, e.g. theClock.Now() == time.Now().
+	theClock clock.Clock = clock.C
+)
+
+// TranslateCPEToCVE maps the CVEs found in NVD archive files in the
+// vulnerabilities database folder to software CPEs in the fleet database.
+// If collectRecentVulns is true, it also returns a mapping of recent CVEs
+// to a list of CPEs affected by the CVE, otherwise that map is nil.
 func TranslateCPEToCVE(
 	ctx context.Context,
 	ds fleet.Datastore,
 	vulnPath string,
 	logger kitlog.Logger,
 	config config.FleetConfig,
-) error {
+	collectRecentVulns bool,
+) (map[string][]string, error) {
 	err := SyncCVEData(vulnPath, config)
 	if err != nil {
-		return err
+		return nil, err
 	}

 	var files []string
 	err = filepath.Walk(vulnPath, func(path string, info os.FileInfo, err error) error {
-		if match, err := regexp.MatchString("nvdcve.*\\.gz$", path); !match || err != nil {
+		if match := rxNVDCVEArchive.MatchString(path); !match {
 			return nil
 		}
 		files = append(files, path)
 		return nil
 	})
 	if err != nil {
-		return err
+		return nil, err
+	}
+
+	if len(files) == 0 {
+		return nil, nil
 	}

 	cpeList, err := ds.AllCPEs(ctx)
 	if err != nil {
-		return err
+		return nil, err
 	}

 	cpes := make([]*wfn.Attributes, 0, len(cpeList))
 	for _, uri := range cpeList {
 		attr, err := wfn.Parse(uri)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		cpes = append(cpes, attr)
 	}

 	if len(cpes) == 0 {
-		return nil
+		return nil, nil
 	}

+	var recentVulns map[string][]string
+	if collectRecentVulns {
+		recentVulns = make(map[string][]string)
+	}
 	for _, file := range files {
-		err := checkCVEs(ctx, ds, logger, cpes, file)
+		err := checkCVEs(ctx, ds, logger, cpes, file, recentVulns)
 		if err != nil {
-			return err
+			return nil, err
 		}
 	}

-	return nil
+	return recentVulns, nil
 }

-func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, cpes []*wfn.Attributes, files ...string) error {
-	dict, err := cvefeed.LoadJSONDictionary(files...)
+func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger,
+	cpes []*wfn.Attributes, file string, recentVulns map[string][]string) error {
+
+	dict, err := cvefeed.LoadJSONDictionary(file)
 	if err != nil {
 		return err
 	}
@@ -111,9 +142,10 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, cp
 	//cache.Idx = cvefeed.NewIndex(dict)

 	cpeCh := make(chan *wfn.Attributes)
+	collectVulns := recentVulns != nil

 	var wg sync.WaitGroup
-
+	var mu sync.Mutex
 	for i := 0; i < runtime.NumCPU(); i++ {
 		wg.Add(1)
 		goRoutineKey := i
@@ -136,10 +168,12 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, cp
 						if ml == 0 {
 							continue
 						}
+
+						cveID := matches.CVE.ID()
 						matchingCPEs := make([]string, 0, ml)
 						for _, attr := range matches.CPEs {
 							if attr == nil {
-								level.Error(logger).Log("matches nil CPE", matches.CVE.ID())
+								level.Error(logger).Log("matches nil CPE", cveID)
 								continue
 							}
 							cpe := attr.BindToFmtString()
@@ -148,9 +182,45 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, cp
 							}
 							matchingCPEs = append(matchingCPEs, cpe)
 						}
-						err = ds.InsertCVEForCPE(ctx, matches.CVE.ID(), matchingCPEs)
+
+						newCount, err := ds.InsertCVEForCPE(ctx, cveID, matchingCPEs)
 						if err != nil {
 							level.Error(logger).Log("cpe processing", "error", "err", err)
+							continue // do not report a recent vuln that failed to be inserted in the DB
+						}
+
+						// collect as recent vuln only if newCount > 0, otherwise we would send
+						// webhook requests for the same vulnerability over and over again until
+						// it is older than 2 days.
+						if collectVulns && newCount > 0 {
+							vuln, ok := matches.CVE.(*feednvd.Vuln)
+							if !ok {
+								level.Error(logger).Log("recent vuln", "unexpected type for Vuln interface", "cve", cveID,
+									"type", fmt.Sprintf("%T", matches.CVE))
+								continue
+							}
+
+							rawPubDate := vuln.Schema().PublishedDate
+							if rawPubDate == "" {
+								level.Error(logger).Log("recent vuln", "empty published date", "cve", cveID)
+								continue
+							}
+
+							pubDate, err := time.Parse(publishedDateFmt, rawPubDate)
+							if err != nil {
+								level.Error(logger).Log("recent vuln", "unexpected published date format", "cve", cveID,
+									"published_date", rawPubDate, "err", err)
+								continue
+							}
+
+							// the second condition should only affect tests - to ignore pubDates in the future
+							// when using a mocked current clock. When using the real clock, the published date
+							// should always be in the past.
+							if theClock.Since(pubDate) <= recentVulnMaxAge && theClock.Now().After(pubDate) {
+								mu.Lock()
+								recentVulns[cveID] = append(recentVulns[cveID], matchingCPEs...)
+								mu.Unlock()
+							}
 						}
 					}
 				case <-ctx.Done():
@@ -170,6 +240,5 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, cp
 	level.Debug(logger).Log("pushing cpes", "done")

 	wg.Wait()
-
 	return nil
 }
@@ -12,10 +12,13 @@ import (
 	"strings"
 	"sync"
 	"testing"
+	"time"

+	"github.com/WatchBeam/clock"
 	"github.com/fleetdm/fleet/v4/server/config"
 	"github.com/fleetdm/fleet/v4/server/mock"
 	kitlog "github.com/go-kit/kit/log"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

@@ -26,7 +29,7 @@ var cvetests = []struct {
 	{"cpe:2.3:a:1password:1password:3.9.9:*:*:*:*:*:*:*", "CVE-2012-6369"},
 }

-func PrintMemUsage() {
+func printMemUsage() {
 	var m runtime.MemStats
 	runtime.ReadMemStats(&m)
 	// For info on each, see: https://golang.org/pkg/runtime/#MemStats
@@ -50,6 +53,12 @@ func TestTranslateCPEToCVE(t *testing.T) {
 	ds := new(mock.Store)
 	ctx := context.Background()

+	// download the CVEs once for all sub-tests, and then disable syncing
+	cfg := config.FleetConfig{}
+	err := SyncCVEData(tempDir, cfg)
+	require.NoError(t, err)
+	cfg.Vulnerabilities.DisableDataSync = true
+
 	for _, tt := range cvetests {
 		t.Run(tt.cpe, func(t *testing.T) {
 			ds.AllCPEsFunc = func(ctx context.Context) ([]string, error) {
@@ -59,23 +68,70 @@ func TestTranslateCPEToCVE(t *testing.T) {
 			cveLock := &sync.Mutex{}
 			cveToCPEs := make(map[string][]string)
 			var cvesFound []string
-			ds.InsertCVEForCPEFunc = func(ctx context.Context, cve string, cpes []string) error {
+			ds.InsertCVEForCPEFunc = func(ctx context.Context, cve string, cpes []string) (int64, error) {
 				cveLock.Lock()
 				defer cveLock.Unlock()
 				cveToCPEs[cve] = cpes
 				cvesFound = append(cvesFound, cve)
-				return nil
+				return 0, nil
 			}

-			err := TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewLogfmtLogger(os.Stdout), config.FleetConfig{})
+			_, err := TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewLogfmtLogger(os.Stdout), cfg, false)
 			require.NoError(t, err)

-			PrintMemUsage()
+			printMemUsage()

 			require.Equal(t, []string{tt.cve}, cvesFound)
 			require.Equal(t, []string{tt.cpe}, cveToCPEs[tt.cve])
 		})
 	}
+
+	t.Run("recent_vulns", func(t *testing.T) {
+		googleChromeCPE := "cpe:2.3:a:google:chrome:-:*:*:*:*:*:*:*"
+		mozillaFirefoxCPE := "cpe:2.3:a:mozilla:firefox:-:*:*:*:*:*:*:*"
+		curlCPE := "cpe:2.3:a:haxx:curl:-:*:*:*:*:*:*:*"
+
+		// consider recent vulnerabilities to be anything published in 2018
+		theClock = clock.NewMockClock(time.Date(2019, 01, 01, 0, 0, 0, 0, time.UTC))
+		oldMaxAge := recentVulnMaxAge
+		recentVulnMaxAge = 365 * 24 * time.Hour
+		defer func() { recentVulnMaxAge = oldMaxAge; theClock = clock.C }()
+
+		ds.AllCPEsFunc = func(ctx context.Context) ([]string, error) {
+			return []string{googleChromeCPE, mozillaFirefoxCPE, curlCPE}, nil
+		}
+
+		ds.InsertCVEForCPEFunc = func(ctx context.Context, cve string, cpes []string) (int64, error) {
+			return 1, nil
+		}
+		recent, err := TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewNopLogger(), cfg, true)
+		require.NoError(t, err)
+
+		byCPE := make(map[string]int)
+		for _, cpes := range recent {
+			for _, cpe := range cpes {
+				byCPE[cpe]++
+			}
+		}
+
+		// even if it's somewhat far in the past, I've seen the exact numbers
+		// change a bit between runs with different downloads, so allow for a bit
+		// of wiggle room.
+		assert.Greater(t, byCPE[googleChromeCPE], 150, "google chrome CVEs")
+		assert.Greater(t, byCPE[mozillaFirefoxCPE], 280, "mozilla firefox CVEs")
+		assert.Greater(t, byCPE[curlCPE], 10, "curl CVEs")
+
+		// call it again but now return 0 from this call, simulating CVE-CPE pairs
+		// that already existed in the DB.
+		ds.InsertCVEForCPEFunc = func(ctx context.Context, cve string, cpes []string) (int64, error) {
+			return 0, nil
+		}
+		recent, err = TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewNopLogger(), cfg, true)
+		require.NoError(t, err)
+
+		// no recent vulnerability should be reported
+		assert.Len(t, recent, 0)
+	})
 }

 func TestSyncsCVEFromURL(t *testing.T) {