Trigger webhooks for recently published vulnerabilities (#3941)

changes/issue-3050-trigger-vulnerabilities-webhook

@@ -0,0 +1 @@
+* Support triggering a webhook for newly detected vulnerabilities with a list of affected hosts

cmd/fleet/serve.go

@@ -706,33 +706,42 @@ func cronVulnerabilities(
 		}
 
 		if !vulnDisabled {
-			checkVulnerabilities(ctx, ds, logger, vulnPath, config)
+			recentVulns := checkVulnerabilities(ctx, ds, logger, vulnPath, config, appConfig.WebhookSettings.VulnerabilitiesWebhook)
+			if len(recentVulns) > 0 {
+				if err := webhooks.TriggerVulnerabilitiesWebhook(ctx, ds, kitlog.With(logger, "webhook", "vulnerabilities"),
+					recentVulns, appConfig, time.Now()); err != nil {
+
+					level.Error(logger).Log("err", "triggering vulnerabilities webhook", "details", err)
+					sentry.CaptureException(err)
+				}
+			}
 		}
 
 		if err := ds.CalculateHostsPerSoftware(ctx, time.Now()); err != nil {
 			level.Error(logger).Log("msg", "calculating hosts count per software", "err", err)
 			sentry.CaptureException(err)
-			continue
 		}
 
 		level.Debug(logger).Log("loop", "done")
 	}
 }
 
-func checkVulnerabilities(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, vulnPath string, config config.FleetConfig) {
+func checkVulnerabilities(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger,
+	vulnPath string, config config.FleetConfig, vulnWebhookCfg fleet.VulnerabilitiesWebhookSettings) map[string][]string {
 	err := vulnerabilities.TranslateSoftwareToCPE(ctx, ds, vulnPath, logger, config)
 	if err != nil {
 		level.Error(logger).Log("msg", "analyzing vulnerable software: Software->CPE", "err", err)
 		sentry.CaptureException(err)
-		return
+		return nil
 	}
 
-	err = vulnerabilities.TranslateCPEToCVE(ctx, ds, vulnPath, logger, config)
+	recentVulns, err := vulnerabilities.TranslateCPEToCVE(ctx, ds, vulnPath, logger, config, vulnWebhookCfg.Enable)
 	if err != nil {
 		level.Error(logger).Log("msg", "analyzing vulnerable software: CPE->CVE", "err", err)
 		sentry.CaptureException(err)
-		return
+		return nil
 	}
+	return recentVulns
 }
 
 func cronWebhooks(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, identifier string, failingPoliciesSet fleet.FailingPolicySet) {

docs/01-Using-Fleet/configuration-files/README.md

@@ -449,6 +449,16 @@ The following options allow the configuration of a webhook that will be triggere
 - `webhook_settings.failing_policies_webhook.policy_ids`: the IDs of the policies for which the webhook will be enabled.
 - `webhook_settings.failing_policies_webhook.host_batch_size`: Maximum number of hosts to batch on POST requests. A value of `0`, the default, means no batching, all hosts failing a policy will be sent on one POST request.
 
+##### Recent Vulnerabilities
+
+The following options allow the configuration of a webhook that will be triggered if recently published vulnerabilities are detected and there are affected hosts. A vulnerability is considered recent if it has been published in the last 2 days (based on the National Vulnerability Database, NVD).
+
+- `webhook_settings.vulnerabilities_webhook.enable_vulnerabilities_webhook`: true or false. Defines whether to enable the vulnerabilities webhook.
+- `webhook_settings.vulnerabilities_webhook.destination_url`: the URL to POST to when the condition for the webhook triggers.
+- `webhook_settings.vulnerabilities_webhook.host_batch_size`: Maximum number of hosts to batch on POST requests. A value of `0`, the default, means no batching, all hosts affected will be sent on one POST request.
+
+Note that the recent vulnerabilities webhook is not checked at `webhook_settings.interval` like other webhooks - it is checked as part of the vulnerability processing and runs at the `vulnerabilities.periodicity` interval specified in the fleet configuration.
+
 #### Debug host
 
 There's a lot of information coming from hosts, but it's sometimes useful to see exactly what a host is returning in order

server/datastore/mysql/migrations/tables/20220201084510_AddSoftwareCpeIndex.go

@@ -0,0 +1,22 @@
+package tables
+
+import (
+	"database/sql"
+
+	"github.com/pkg/errors"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20220201084510, Down_20220201084510)
+}
+
+func Up_20220201084510(tx *sql.Tx) error {
+	if _, err := tx.Exec(`CREATE INDEX software_cpe_cpe_idx ON software_cpe(cpe);`); err != nil {
+		return errors.Wrap(err, "creating software_cpe index")
+	}
+	return nil
+}
+
+func Down_20220201084510(tx *sql.Tx) error {
+	return nil
+}

server/datastore/mysql/software.go

@@ -494,18 +494,22 @@ func (d *Datastore) AllCPEs(ctx context.Context) ([]string, error) {
 	return cpes, nil
 }
 
-func (d *Datastore) InsertCVEForCPE(ctx context.Context, cve string, cpes []string) error {
+// InsertCVEForCPE inserts the cve into software_cve, linking it to all the
+// provided cpes. It returns the number of new rows inserted or an error. If
+// the CVE already existed for all CPEs, it would return 0, nil.
+func (d *Datastore) InsertCVEForCPE(ctx context.Context, cve string, cpes []string) (int64, error) {
 	values := strings.TrimSuffix(strings.Repeat("((SELECT id FROM software_cpe WHERE cpe=?),?),", len(cpes)), ",")
 	sql := fmt.Sprintf(`INSERT IGNORE INTO software_cve (cpe_id, cve) VALUES %s`, values)
 	var args []interface{}
 	for _, cpe := range cpes {
 		args = append(args, cpe, cve)
 	}
-	_, err := d.writer.ExecContext(ctx, sql, args...)
+	res, err := d.writer.ExecContext(ctx, sql, args...)
 	if err != nil {
-		return ctxerr.Wrap(ctx, err, "insert software cve")
+		return 0, ctxerr.Wrap(ctx, err, "insert software cve")
 	}
-	return nil
+	count, _ := res.RowsAffected()
+	return count, nil
 }
 
 func (d *Datastore) ListSoftware(ctx context.Context, opt fleet.SoftwareListOptions) ([]fleet.Software, error) {
@@ -652,3 +656,37 @@ func (d *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt tim
 
 	return nil
 }
+
+// HostsByCPEs returns a list of all hosts that have the software corresponding
+// to at least one of the CPEs installed. It returns a minimal represention of
+// matching hosts.
+func (d *Datastore) HostsByCPEs(ctx context.Context, cpes []string) ([]*fleet.CPEHost, error) {
+	queryStmt := `
+    SELECT
+      h.id,
+      h.hostname
+    FROM
+      hosts h
+    INNER JOIN
+      host_software hs
+    ON
+      h.id = hs.host_id
+    INNER JOIN
+      software_cpe scp
+    ON
+      hs.software_id = scp.software_id
+    WHERE
+      scp.cpe IN (?)
+    ORDER BY
+      h.id`
+
+	stmt, args, err := sqlx.In(queryStmt, cpes)
+	if err != nil {
+		return nil, ctxerr.Wrap(ctx, err, "building query args")
+	}
+	var hosts []*fleet.CPEHost
+	if err := sqlx.SelectContext(ctx, d.reader, &hosts, stmt, args...); err != nil {
+		return nil, ctxerr.Wrap(ctx, err, "select hosts by cpes")
+	}
+	return hosts, nil
+}

server/datastore/mysql/software_test.go

@@ -202,7 +202,14 @@ func testSoftwareInsertCVEs(t *testing.T, ds *Datastore) {
 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host))
 
 	require.NoError(t, ds.AddCPEForSoftware(context.Background(), host.Software[0], "somecpe"))
-	require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"}))
+	count, err := ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})
+	require.NoError(t, err)
+	assert.Equal(t, int64(1), count)
+
+	// run again for the same CPE, should not create any new row
+	count, err = ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), count)
 }
 
 func testSoftwareHostDuplicates(t *testing.T, ds *Datastore) {
@@ -250,8 +257,10 @@ func testSoftwareLoadVulnerabilities(t *testing.T, ds *Datastore) {
 
 	require.NoError(t, ds.AddCPEForSoftware(context.Background(), host.Software[0], "somecpe"))
 	require.NoError(t, ds.AddCPEForSoftware(context.Background(), host.Software[1], "someothercpewithoutvulns"))
-	require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"}))
-	require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-321-321-321", []string{"somecpe"}))
+	_, err := ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})
+	require.NoError(t, err)
+	_, err = ds.InsertCVEForCPE(context.Background(), "cve-321-321-321", []string{"somecpe"})
+	require.NoError(t, err)
 
 	require.NoError(t, ds.LoadHostSoftware(context.Background(), host))
 
@@ -387,8 +396,10 @@ func testSoftwareList(t *testing.T, ds *Datastore) {
 	})
 	require.NoError(t, ds.AddCPEForSoftware(context.Background(), host1.Software[0], "somecpe"))
 	require.NoError(t, ds.AddCPEForSoftware(context.Background(), host1.Software[1], "someothercpewithoutvulns"))
-	require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-321-432-543", []string{"somecpe"}))
-	require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-333-444-555", []string{"somecpe"}))
+	_, err := ds.InsertCVEForCPE(context.Background(), "cve-321-432-543", []string{"somecpe"})
+	require.NoError(t, err)
+	_, err = ds.InsertCVEForCPE(context.Background(), "cve-333-444-555", []string{"somecpe"})
+	require.NoError(t, err)
 
 	foo001 := fleet.Software{
 		Name: "foo", Version: "0.0.1", Source: "chrome_extensions", GenerateCPE: "somecpe",

server/fleet/datastore.go

@@ -328,9 +328,10 @@ type Datastore interface {
 	AllSoftwareWithoutCPEIterator(ctx context.Context) (SoftwareIterator, error)
 	AddCPEForSoftware(ctx context.Context, software Software, cpe string) error
 	AllCPEs(ctx context.Context) ([]string, error)
-	InsertCVEForCPE(ctx context.Context, cve string, cpes []string) error
+	InsertCVEForCPE(ctx context.Context, cve string, cpes []string) (int64, error)
 	SoftwareByID(ctx context.Context, id uint) (*Software, error)
 	CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error
+	HostsByCPEs(ctx context.Context, cpes []string) ([]*CPEHost, error)
 
 	///////////////////////////////////////////////////////////////////////////////
 	// ActivitiesStore

server/fleet/hosts.go

@@ -286,3 +286,10 @@ type AggregatedMacadminsData struct {
 	MunkiVersions []AggregatedMunkiVersion `json:"munki_versions"`
 	MDMStatus     AggregatedMDMStatus      `json:"mobile_device_management_enrollment_status"`
 }
+
+// CPEHost is a minimal host representation returned when querying hosts by
+// CPE.
+type CPEHost struct {
+	ID       uint   `json:"id" db:"id"`
+	Hostname string `json:"hostname" db:"hostname"`
+}

server/mock/datastore_mock.go

@@ -268,12 +268,14 @@ type AddCPEForSoftwareFunc func(ctx context.Context, software fleet.Software, cp
 
 type AllCPEsFunc func(ctx context.Context) ([]string, error)
 
-type InsertCVEForCPEFunc func(ctx context.Context, cve string, cpes []string) error
+type InsertCVEForCPEFunc func(ctx context.Context, cve string, cpes []string) (int64, error)
 
 type SoftwareByIDFunc func(ctx context.Context, id uint) (*fleet.Software, error)
 
 type CalculateHostsPerSoftwareFunc func(ctx context.Context, updatedAt time.Time) error
 
+type HostsByCPEsFunc func(ctx context.Context, cpes []string) ([]*fleet.CPEHost, error)
+
 type NewActivityFunc func(ctx context.Context, user *fleet.User, activityType string, details *map[string]interface{}) error
 
 type ListActivitiesFunc func(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Activity, error)
@@ -762,6 +764,9 @@ type DataStore struct {
 	CalculateHostsPerSoftwareFunc        CalculateHostsPerSoftwareFunc
 	CalculateHostsPerSoftwareFuncInvoked bool
 
+	HostsByCPEsFunc        HostsByCPEsFunc
+	HostsByCPEsFuncInvoked bool
+
 	NewActivityFunc        NewActivityFunc
 	NewActivityFuncInvoked bool
 
@@ -1544,7 +1549,7 @@ func (s *DataStore) AllCPEs(ctx context.Context) ([]string, error) {
 	return s.AllCPEsFunc(ctx)
 }
 
-func (s *DataStore) InsertCVEForCPE(ctx context.Context, cve string, cpes []string) error {
+func (s *DataStore) InsertCVEForCPE(ctx context.Context, cve string, cpes []string) (int64, error) {
 	s.InsertCVEForCPEFuncInvoked = true
 	return s.InsertCVEForCPEFunc(ctx, cve, cpes)
 }
@@ -1559,6 +1564,11 @@ func (s *DataStore) CalculateHostsPerSoftware(ctx context.Context, updatedAt tim
 	return s.CalculateHostsPerSoftwareFunc(ctx, updatedAt)
 }
 
+func (s *DataStore) HostsByCPEs(ctx context.Context, cpes []string) ([]*fleet.CPEHost, error) {
+	s.HostsByCPEsFuncInvoked = true
+	return s.HostsByCPEsFunc(ctx, cpes)
+}
+
 func (s *DataStore) NewActivity(ctx context.Context, user *fleet.User, activityType string, details *map[string]interface{}) error {
 	s.NewActivityFuncInvoked = true
 	return s.NewActivityFunc(ctx, user, activityType, details)

server/service/integration_core_test.go

@@ -374,7 +374,8 @@ func (s *integrationTestSuite) TestVulnerableSoftware() {
 	}
 
 	require.NoError(t, s.ds.AddCPEForSoftware(context.Background(), soft1, "somecpe"))
-	require.NoError(t, s.ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"}))
+	_, err = s.ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})
+	require.NoError(t, err)
 
 	resp := s.Do("GET", fmt.Sprintf("/api/v1/fleet/hosts/%d", host.ID), nil, http.StatusOK)
 	bodyBytes, err := ioutil.ReadAll(resp.Body)

server/vulnerabilities/cve.go

@@ -11,7 +11,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/WatchBeam/clock"
 	"github.com/facebookincubator/nvdtools/cvefeed"
+	feednvd "github.com/facebookincubator/nvdtools/cvefeed/nvd"
 	"github.com/facebookincubator/nvdtools/providers/nvd"
 	"github.com/facebookincubator/nvdtools/wfn"
 	"github.com/fleetdm/fleet/v4/server/config"
@@ -49,60 +51,89 @@ func SyncCVEData(vulnPath string, config config.FleetConfig) error {
 	return dfs.Do(ctx)
 }
 
+const publishedDateFmt = "2006-01-02T15:04Z" // not quite RFC3339
+
+var (
+	rxNVDCVEArchive = regexp.MustCompile(`nvdcve.*\.gz$`)
+
+	// max age to be considered a recent vulnerability (relative to NVD's published date)
+	// (a var to be able to change in tests)
+	recentVulnMaxAge = 2 * 24 * time.Hour
+
+	// this allows mocking the time package for tests, by default it is equivalent
+	// to the time functions, e.g. theClock.Now() == time.Now().
+	theClock clock.Clock = clock.C
+)
+
+// TranslateCPEToCVE maps the CVEs found in NVD archive files in the
+// vulnerabilities database folder to software CPEs in the fleet database.
+// If collectRecentVulns is true, it also returns a mapping of recent CVEs
+// to a list of CPEs affected by the CVE, otherwise that map is nil.
 func TranslateCPEToCVE(
 	ctx context.Context,
 	ds fleet.Datastore,
 	vulnPath string,
 	logger kitlog.Logger,
 	config config.FleetConfig,
-) error {
+	collectRecentVulns bool,
+) (map[string][]string, error) {
 	err := SyncCVEData(vulnPath, config)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	var files []string
 	err = filepath.Walk(vulnPath, func(path string, info os.FileInfo, err error) error {
-		if match, err := regexp.MatchString("nvdcve.*\\.gz$", path); !match || err != nil {
+		if match := rxNVDCVEArchive.MatchString(path); !match {
 			return nil
 		}
 		files = append(files, path)
 		return nil
 	})
 	if err != nil {
-		return err
+		return nil, err
+	}
+
+	if len(files) == 0 {
+		return nil, nil
 	}
 
 	cpeList, err := ds.AllCPEs(ctx)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	cpes := make([]*wfn.Attributes, 0, len(cpeList))
 	for _, uri := range cpeList {
 		attr, err := wfn.Parse(uri)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		cpes = append(cpes, attr)
 	}
 
 	if len(cpes) == 0 {
-		return nil
+		return nil, nil
 	}
 
+	var recentVulns map[string][]string
+	if collectRecentVulns {
+		recentVulns = make(map[string][]string)
+	}
 	for _, file := range files {
-		err := checkCVEs(ctx, ds, logger, cpes, file)
+		err := checkCVEs(ctx, ds, logger, cpes, file, recentVulns)
 		if err != nil {
-			return err
+			return nil, err
 		}
 	}
 
-	return nil
+	return recentVulns, nil
 }
 
-func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, cpes []*wfn.Attributes, files ...string) error {
-	dict, err := cvefeed.LoadJSONDictionary(files...)
+func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger,
+	cpes []*wfn.Attributes, file string, recentVulns map[string][]string) error {
+
+	dict, err := cvefeed.LoadJSONDictionary(file)
 	if err != nil {
 		return err
 	}
@@ -111,9 +142,10 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, cp
 	//cache.Idx = cvefeed.NewIndex(dict)
 
 	cpeCh := make(chan *wfn.Attributes)
+	collectVulns := recentVulns != nil
 
 	var wg sync.WaitGroup
-
+	var mu sync.Mutex
 	for i := 0; i < runtime.NumCPU(); i++ {
 		wg.Add(1)
 		goRoutineKey := i
@@ -136,10 +168,12 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, cp
 						if ml == 0 {
 							continue
 						}
+
+						cveID := matches.CVE.ID()
 						matchingCPEs := make([]string, 0, ml)
 						for _, attr := range matches.CPEs {
 							if attr == nil {
-								level.Error(logger).Log("matches nil CPE", matches.CVE.ID())
+								level.Error(logger).Log("matches nil CPE", cveID)
 								continue
 							}
 							cpe := attr.BindToFmtString()
@@ -148,9 +182,45 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, cp
 							}
 							matchingCPEs = append(matchingCPEs, cpe)
 						}
-						err = ds.InsertCVEForCPE(ctx, matches.CVE.ID(), matchingCPEs)
+
+						newCount, err := ds.InsertCVEForCPE(ctx, cveID, matchingCPEs)
 						if err != nil {
 							level.Error(logger).Log("cpe processing", "error", "err", err)
+							continue // do not report a recent vuln that failed to be inserted in the DB
+						}
+
+						// collect as recent vuln only if newCount > 0, otherwise we would send
+						// webhook requests for the same vulnerability over and over again until
+						// it is older than 2 days.
+						if collectVulns && newCount > 0 {
+							vuln, ok := matches.CVE.(*feednvd.Vuln)
+							if !ok {
+								level.Error(logger).Log("recent vuln", "unexpected type for Vuln interface", "cve", cveID,
+									"type", fmt.Sprintf("%T", matches.CVE))
+								continue
+							}
+
+							rawPubDate := vuln.Schema().PublishedDate
+							if rawPubDate == "" {
+								level.Error(logger).Log("recent vuln", "empty published date", "cve", cveID)
+								continue
+							}
+
+							pubDate, err := time.Parse(publishedDateFmt, rawPubDate)
+							if err != nil {
+								level.Error(logger).Log("recent vuln", "unexpected published date format", "cve", cveID,
+									"published_date", rawPubDate, "err", err)
+								continue
+							}
+
+							// the second condition should only affect tests - to ignore pubDates in the future
+							// when using a mocked current clock. When using the real clock, the published date
+							// should always be in the past.
+							if theClock.Since(pubDate) <= recentVulnMaxAge && theClock.Now().After(pubDate) {
+								mu.Lock()
+								recentVulns[cveID] = append(recentVulns[cveID], matchingCPEs...)
+								mu.Unlock()
+							}
 						}
 					}
 				case <-ctx.Done():
@@ -170,6 +240,5 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, cp
 	level.Debug(logger).Log("pushing cpes", "done")
 
 	wg.Wait()
-
 	return nil
 }

server/vulnerabilities/cve_test.go

@@ -12,10 +12,13 @@ import (
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
+	"github.com/WatchBeam/clock"
 	"github.com/fleetdm/fleet/v4/server/config"
 	"github.com/fleetdm/fleet/v4/server/mock"
 	kitlog "github.com/go-kit/kit/log"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -26,7 +29,7 @@ var cvetests = []struct {
 	{"cpe:2.3:a:1password:1password:3.9.9:*:*:*:*:*:*:*", "CVE-2012-6369"},
 }
 
-func PrintMemUsage() {
+func printMemUsage() {
 	var m runtime.MemStats
 	runtime.ReadMemStats(&m)
 	// For info on each, see: https://golang.org/pkg/runtime/#MemStats
@@ -50,6 +53,12 @@ func TestTranslateCPEToCVE(t *testing.T) {
 	ds := new(mock.Store)
 	ctx := context.Background()
 
+	// download the CVEs once for all sub-tests, and then disable syncing
+	cfg := config.FleetConfig{}
+	err := SyncCVEData(tempDir, cfg)
+	require.NoError(t, err)
+	cfg.Vulnerabilities.DisableDataSync = true
+
 	for _, tt := range cvetests {
 		t.Run(tt.cpe, func(t *testing.T) {
 			ds.AllCPEsFunc = func(ctx context.Context) ([]string, error) {
@@ -59,23 +68,70 @@ func TestTranslateCPEToCVE(t *testing.T) {
 			cveLock := &sync.Mutex{}
 			cveToCPEs := make(map[string][]string)
 			var cvesFound []string
-			ds.InsertCVEForCPEFunc = func(ctx context.Context, cve string, cpes []string) error {
+			ds.InsertCVEForCPEFunc = func(ctx context.Context, cve string, cpes []string) (int64, error) {
 				cveLock.Lock()
 				defer cveLock.Unlock()
 				cveToCPEs[cve] = cpes
 				cvesFound = append(cvesFound, cve)
-				return nil
+				return 0, nil
 			}
 
-			err := TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewLogfmtLogger(os.Stdout), config.FleetConfig{})
+			_, err := TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewLogfmtLogger(os.Stdout), cfg, false)
 			require.NoError(t, err)
 
-			PrintMemUsage()
+			printMemUsage()
 
 			require.Equal(t, []string{tt.cve}, cvesFound)
 			require.Equal(t, []string{tt.cpe}, cveToCPEs[tt.cve])
 		})
 	}
+
+	t.Run("recent_vulns", func(t *testing.T) {
+		googleChromeCPE := "cpe:2.3:a:google:chrome:-:*:*:*:*:*:*:*"
+		mozillaFirefoxCPE := "cpe:2.3:a:mozilla:firefox:-:*:*:*:*:*:*:*"
+		curlCPE := "cpe:2.3:a:haxx:curl:-:*:*:*:*:*:*:*"
+
+		// consider recent vulnerabilities to be anything published in 2018
+		theClock = clock.NewMockClock(time.Date(2019, 01, 01, 0, 0, 0, 0, time.UTC))
+		oldMaxAge := recentVulnMaxAge
+		recentVulnMaxAge = 365 * 24 * time.Hour
+		defer func() { recentVulnMaxAge = oldMaxAge; theClock = clock.C }()
+
+		ds.AllCPEsFunc = func(ctx context.Context) ([]string, error) {
+			return []string{googleChromeCPE, mozillaFirefoxCPE, curlCPE}, nil
+		}
+
+		ds.InsertCVEForCPEFunc = func(ctx context.Context, cve string, cpes []string) (int64, error) {
+			return 1, nil
+		}
+		recent, err := TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewNopLogger(), cfg, true)
+		require.NoError(t, err)
+
+		byCPE := make(map[string]int)
+		for _, cpes := range recent {
+			for _, cpe := range cpes {
+				byCPE[cpe]++
+			}
+		}
+
+		// even if it's somewhat far in the past, I've seen the exact numbers
+		// change a bit between runs with different downloads, so allow for a bit
+		// of wiggle room.
+		assert.Greater(t, byCPE[googleChromeCPE], 150, "google chrome CVEs")
+		assert.Greater(t, byCPE[mozillaFirefoxCPE], 280, "mozilla firefox CVEs")
+		assert.Greater(t, byCPE[curlCPE], 10, "curl CVEs")
+
+		// call it again but now return 0 from this call, simulating CVE-CPE pairs
+		// that already existed in the DB.
+		ds.InsertCVEForCPEFunc = func(ctx context.Context, cve string, cpes []string) (int64, error) {
+			return 0, nil
+		}
+		recent, err = TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewNopLogger(), cfg, true)
+		require.NoError(t, err)
+
+		// no recent vulnerability should be reported
+		assert.Len(t, recent, 0)
+	})
 }
 
 func TestSyncsCVEFromURL(t *testing.T) {

server/webhooks/vulnerabilities.go

@@ -0,0 +1,94 @@
+package webhooks
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"path"
+	"strconv"
+	"time"
+
+	"github.com/fleetdm/fleet/v4/server"
+	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	kitlog "github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+)
+
+// TriggerVulnerabilitiesWebhook performs the webhook requests for vulnerabilities.
+func TriggerVulnerabilitiesWebhook(
+	ctx context.Context,
+	ds fleet.Datastore,
+	logger kitlog.Logger,
+	recentVulns map[string][]string,
+	appConfig *fleet.AppConfig,
+	now time.Time,
+) error {
+	vulnConfig := appConfig.WebhookSettings.VulnerabilitiesWebhook
+	if !vulnConfig.Enable {
+		return nil
+	}
+
+	level.Debug(logger).Log("enabled", "true", "recentVulns", len(recentVulns))
+
+	serverURL, err := url.Parse(appConfig.ServerSettings.ServerURL)
+	if err != nil {
+		return ctxerr.Wrap(ctx, err, "invalid server url")
+	}
+
+	targetURL := vulnConfig.DestinationURL
+	batchSize := vulnConfig.HostBatchSize
+
+	for cve, cpes := range recentVulns {
+		hosts, err := ds.HostsByCPEs(ctx, cpes)
+		if err != nil {
+			return ctxerr.Wrap(ctx, err, "get hosts by CPE")
+		}
+
+		for len(hosts) > 0 {
+			limit := len(hosts)
+			if batchSize > 0 && len(hosts) > batchSize {
+				limit = batchSize
+			}
+			if err := sendVulnerabilityHostBatch(ctx, targetURL, cve, serverURL, hosts[:limit], now); err != nil {
+				return ctxerr.Wrap(ctx, err, "send vulnerability host batch")
+			}
+			hosts = hosts[limit:]
+		}
+	}
+
+	return nil
+}
+
+type vulnHostPayload struct {
+	ID       uint   `json:"id"`
+	Hostname string `json:"hostname"`
+	URL      string `json:"url"`
+}
+
+func sendVulnerabilityHostBatch(ctx context.Context, targetURL, cve string, hostBaseURL *url.URL, hosts []*fleet.CPEHost, now time.Time) error {
+	shortHosts := make([]*vulnHostPayload, len(hosts))
+	for i, h := range hosts {
+		hostURL := *hostBaseURL
+		hostURL.Path = path.Join(hostURL.Path, "hosts", strconv.Itoa(int(h.ID)))
+		shortHosts[i] = &vulnHostPayload{
+			ID:       h.ID,
+			Hostname: h.Hostname,
+			URL:      hostURL.String(),
+		}
+	}
+
+	payload := map[string]interface{}{
+		"timestamp": now,
+		"vulnerability": map[string]interface{}{
+			"cve":            cve,
+			"details_link":   fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cve),
+			"hosts_affected": shortHosts,
+		},
+	}
+
+	if err := server.PostJSONWithTimeout(ctx, targetURL, &payload); err != nil {
+		return ctxerr.Wrapf(ctx, err, "posting to %s", targetURL)
+	}
+	return nil
+}

server/webhooks/vulnerabilities_test.go

@@ -0,0 +1,151 @@
+package webhooks
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/fleetdm/fleet/v4/server/mock"
+	kitlog "github.com/go-kit/kit/log"
+	"github.com/stretchr/testify/require"
+	"github.com/tj/assert"
+)
+
+func TestTriggerVulnerabilitiesWebhook(t *testing.T) {
+	ctx := context.Background()
+	ds := new(mock.Store)
+	logger := kitlog.NewNopLogger()
+
+	appCfg := &fleet.AppConfig{
+		WebhookSettings: fleet.WebhookSettings{
+			VulnerabilitiesWebhook: fleet.VulnerabilitiesWebhookSettings{
+				Enable:        true,
+				HostBatchSize: 2,
+			},
+		},
+		ServerSettings: fleet.ServerSettings{
+			ServerURL: "https://fleet.example.com",
+		},
+	}
+
+	recentVulns := map[string][]string{
+		"CVE-2012-1234": {"cpe1", "cpe2"},
+	}
+
+	t.Run("disabled", func(t *testing.T) {
+		appCfg := *appCfg
+		appCfg.WebhookSettings.VulnerabilitiesWebhook.Enable = false
+		err := TriggerVulnerabilitiesWebhook(ctx, ds, logger, recentVulns, &appCfg, time.Now())
+		require.NoError(t, err)
+	})
+
+	t.Run("invalid server url", func(t *testing.T) {
+		appCfg := *appCfg
+		appCfg.ServerSettings.ServerURL = ":nope:"
+		err := TriggerVulnerabilitiesWebhook(ctx, ds, logger, recentVulns, &appCfg, time.Now())
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "invalid server")
+	})
+
+	t.Run("empty recent vulns", func(t *testing.T) {
+		err := TriggerVulnerabilitiesWebhook(ctx, ds, logger, nil, appCfg, time.Now())
+		require.NoError(t, err)
+	})
+
+	t.Run("trigger requests", func(t *testing.T) {
+		now := time.Now()
+
+		hosts := []*fleet.CPEHost{
+			{ID: 1, Hostname: "h1"},
+			{ID: 2, Hostname: "h2"},
+			{ID: 3, Hostname: "h3"},
+			{ID: 4, Hostname: "h4"},
+		}
+		jsonH1 := fmt.Sprintf(`{"id":1,"hostname":"h1","url":"%s/hosts/1"}`, appCfg.ServerSettings.ServerURL)
+		jsonH2 := fmt.Sprintf(`{"id":2,"hostname":"h2","url":"%s/hosts/2"}`, appCfg.ServerSettings.ServerURL)
+		jsonH3 := fmt.Sprintf(`{"id":3,"hostname":"h3","url":"%s/hosts/3"}`, appCfg.ServerSettings.ServerURL)
+		jsonH4 := fmt.Sprintf(`{"id":4,"hostname":"h4","url":"%s/hosts/4"}`, appCfg.ServerSettings.ServerURL)
+
+		cves := []string{
+			"CVE-2012-1234",
+			"CVE-2012-4567",
+		}
+		jsonCVE1 := fmt.Sprintf(`{"timestamp":"%s","vulnerability":{"cve":%q,"details_link":"https://nvd.nist.gov/vuln/detail/%[2]s","hosts_affected":`,
+			now.Format(time.RFC3339Nano), cves[0])
+		jsonCVE2 := fmt.Sprintf(`{"timestamp":"%s","vulnerability":{"cve":%q,"details_link":"https://nvd.nist.gov/vuln/detail/%[2]s","hosts_affected":`,
+			now.Format(time.RFC3339Nano), cves[1])
+
+		cases := []struct {
+			name  string
+			vulns map[string][]string
+			hosts []*fleet.CPEHost
+			want  string
+		}{
+			{
+				"1 vuln, 1 host",
+				map[string][]string{cves[0]: {"cpe1"}},
+				hosts[:1],
+				fmt.Sprintf("%s[%s]}}", jsonCVE1, jsonH1),
+			},
+			{
+				"1 vuln, 2 hosts",
+				map[string][]string{cves[0]: {"cpe1"}},
+				hosts[:2],
+				fmt.Sprintf("%s[%s,%s]}}", jsonCVE1, jsonH1, jsonH2),
+			},
+			{
+				"1 vuln, 3 hosts",
+				map[string][]string{cves[0]: {"cpe1"}},
+				hosts[:3],
+				fmt.Sprintf("%s[%s,%s]}}\n%s[%s]}}", jsonCVE1, jsonH1, jsonH2, jsonCVE1, jsonH3), // 2 requests, batch of 2 max
+			},
+			{
+				"1 vuln, 4 hosts",
+				map[string][]string{cves[0]: {"cpe1"}},
+				hosts[:4],
+				fmt.Sprintf("%s[%s,%s]}}\n%s[%s,%s]}}", jsonCVE1, jsonH1, jsonH2, jsonCVE1, jsonH3, jsonH4), // 2 requests, batch of 2 max
+			},
+			{
+				"2 vulns, 1 host each",
+				map[string][]string{cves[0]: {"cpe1"}, cves[1]: {"cpe2"}},
+				hosts[:1],
+				fmt.Sprintf("%s[%s]}}\n%s[%s]}}", jsonCVE1, jsonH1, jsonCVE2, jsonH1),
+			},
+		}
+
+		for _, c := range cases {
+			t.Run(c.name, func(t *testing.T) {
+				var requests []string
+
+				srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					b, err := ioutil.ReadAll(r.Body)
+					assert.NoError(t, err)
+					requests = append(requests, string(b))
+					w.Write(nil)
+				}))
+				defer srv.Close()
+
+				ds.HostsByCPEsFunc = func(ctx context.Context, cpes []string) ([]*fleet.CPEHost, error) {
+					return c.hosts, nil
+				}
+
+				appCfg := *appCfg
+				appCfg.WebhookSettings.VulnerabilitiesWebhook.DestinationURL = srv.URL
+				err := TriggerVulnerabilitiesWebhook(ctx, ds, logger, c.vulns, &appCfg, now)
+				require.NoError(t, err)
+
+				assert.True(t, ds.HostsByCPEsFuncInvoked)
+				ds.HostsByCPEsFuncInvoked = false
+
+				want := strings.Split(c.want, "\n")
+				assert.ElementsMatch(t, want, requests)
+			})
+		}
+	})
+}