From 6e2ba62744bd0b9201495eb84a99affaabedcfc6 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 2 Feb 2022 16:34:37 -0500 Subject: [PATCH] Trigger webhooks for recently published vulnerabilities (#3941) --- ...issue-3050-trigger-vulnerabilities-webhook | 1 + cmd/fleet/serve.go | 21 ++- .../configuration-files/README.md | 10 ++ .../20220201084510_AddSoftwareCpeIndex.go | 22 +++ server/datastore/mysql/schema.sql | 5 +- server/datastore/mysql/software.go | 46 +++++- server/datastore/mysql/software_test.go | 21 ++- server/fleet/datastore.go | 3 +- server/fleet/hosts.go | 7 + server/mock/datastore_mock.go | 14 +- server/service/integration_core_test.go | 3 +- server/vulnerabilities/cve.go | 101 ++++++++++-- server/vulnerabilities/cve_test.go | 66 +++++++- server/webhooks/vulnerabilities.go | 94 +++++++++++ server/webhooks/vulnerabilities_test.go | 151 ++++++++++++++++++ 15 files changed, 523 insertions(+), 42 deletions(-) create mode 100644 changes/issue-3050-trigger-vulnerabilities-webhook create mode 100644 server/datastore/mysql/migrations/tables/20220201084510_AddSoftwareCpeIndex.go create mode 100644 server/webhooks/vulnerabilities.go create mode 100644 server/webhooks/vulnerabilities_test.go diff --git a/changes/issue-3050-trigger-vulnerabilities-webhook b/changes/issue-3050-trigger-vulnerabilities-webhook new file mode 100644 index 0000000000..adc71102fc --- /dev/null +++ b/changes/issue-3050-trigger-vulnerabilities-webhook @@ -0,0 +1 @@ +* Support triggering a webhook for newly detected vulnerabilities with a list of affected hosts diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 2de4e65255..706bde17b2 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -706,33 +706,42 @@ func cronVulnerabilities( } if !vulnDisabled { - checkVulnerabilities(ctx, ds, logger, vulnPath, config) + recentVulns := checkVulnerabilities(ctx, ds, logger, vulnPath, config, appConfig.WebhookSettings.VulnerabilitiesWebhook) + if len(recentVulns) > 0 { + if err := webhooks.TriggerVulnerabilitiesWebhook(ctx, ds, kitlog.With(logger, "webhook", "vulnerabilities"), + recentVulns, appConfig, time.Now()); err != nil { + + level.Error(logger).Log("err", "triggering vulnerabilities webhook", "details", err) + sentry.CaptureException(err) + } + } } if err := ds.CalculateHostsPerSoftware(ctx, time.Now()); err != nil { level.Error(logger).Log("msg", "calculating hosts count per software", "err", err) sentry.CaptureException(err) - continue } level.Debug(logger).Log("loop", "done") } } -func checkVulnerabilities(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, vulnPath string, config config.FleetConfig) { +func checkVulnerabilities(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, + vulnPath string, config config.FleetConfig, vulnWebhookCfg fleet.VulnerabilitiesWebhookSettings) map[string][]string { err := vulnerabilities.TranslateSoftwareToCPE(ctx, ds, vulnPath, logger, config) if err != nil { level.Error(logger).Log("msg", "analyzing vulnerable software: Software->CPE", "err", err) sentry.CaptureException(err) - return + return nil } - err = vulnerabilities.TranslateCPEToCVE(ctx, ds, vulnPath, logger, config) + recentVulns, err := vulnerabilities.TranslateCPEToCVE(ctx, ds, vulnPath, logger, config, vulnWebhookCfg.Enable) if err != nil { level.Error(logger).Log("msg", "analyzing vulnerable software: CPE->CVE", "err", err) sentry.CaptureException(err) - return + return nil } + return recentVulns } func cronWebhooks(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, identifier string, failingPoliciesSet fleet.FailingPolicySet) { diff --git a/docs/01-Using-Fleet/configuration-files/README.md b/docs/01-Using-Fleet/configuration-files/README.md index a870bbb1fb..a5abdb15ff 100644 --- a/docs/01-Using-Fleet/configuration-files/README.md +++ b/docs/01-Using-Fleet/configuration-files/README.md @@ -449,6 +449,16 @@ The following options allow the configuration of a webhook that will be triggere - `webhook_settings.failing_policies_webhook.policy_ids`: the IDs of the policies for which the webhook will be enabled. - `webhook_settings.failing_policies_webhook.host_batch_size`: Maximum number of hosts to batch on POST requests. A value of `0`, the default, means no batching, all hosts failing a policy will be sent on one POST request. +##### Recent Vulnerabilities + +The following options allow the configuration of a webhook that will be triggered if recently published vulnerabilities are detected and there are affected hosts. A vulnerability is considered recent if it has been published in the last 2 days (based on the National Vulnerability Database, NVD). + +- `webhook_settings.vulnerabilities_webhook.enable_vulnerabilities_webhook`: true or false. Defines whether to enable the vulnerabilities webhook. +- `webhook_settings.vulnerabilities_webhook.destination_url`: the URL to POST to when the condition for the webhook triggers. +- `webhook_settings.vulnerabilities_webhook.host_batch_size`: Maximum number of hosts to batch on POST requests. A value of `0`, the default, means no batching, all hosts affected will be sent on one POST request. + +Note that the recent vulnerabilities webhook is not checked at `webhook_settings.interval` like other webhooks - it is checked as part of the vulnerability processing and runs at the `vulnerabilities.periodicity` interval specified in the fleet configuration. + #### Debug host There's a lot of information coming from hosts, but it's sometimes useful to see exactly what a host is returning in order diff --git a/server/datastore/mysql/migrations/tables/20220201084510_AddSoftwareCpeIndex.go b/server/datastore/mysql/migrations/tables/20220201084510_AddSoftwareCpeIndex.go new file mode 100644 index 0000000000..32467110e2 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20220201084510_AddSoftwareCpeIndex.go @@ -0,0 +1,22 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20220201084510, Down_20220201084510) +} + +func Up_20220201084510(tx *sql.Tx) error { + if _, err := tx.Exec(`CREATE INDEX software_cpe_cpe_idx ON software_cpe(cpe);`); err != nil { + return errors.Wrap(err, "creating software_cpe index") + } + return nil +} + +func Down_20220201084510(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 36c8aa76c3..5bf36e43e1 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -317,9 +317,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=120 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=121 DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `network_interfaces` ( @@ -534,6 +534,7 @@ CREATE TABLE `software_cpe` ( `cpe` varchar(255) NOT NULL, PRIMARY KEY (`id`), KEY `fk_software_cpe_software_id` (`software_id`), + KEY `software_cpe_cpe_idx` (`cpe`), CONSTRAINT `software_cpe_ibfk_1` FOREIGN KEY (`software_id`) REFERENCES `software` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 9b85efd3db..a7642e8230 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -494,18 +494,22 @@ func (d *Datastore) AllCPEs(ctx context.Context) ([]string, error) { return cpes, nil } -func (d *Datastore) InsertCVEForCPE(ctx context.Context, cve string, cpes []string) error { +// InsertCVEForCPE inserts the cve into software_cve, linking it to all the +// provided cpes. It returns the number of new rows inserted or an error. If +// the CVE already existed for all CPEs, it would return 0, nil. +func (d *Datastore) InsertCVEForCPE(ctx context.Context, cve string, cpes []string) (int64, error) { values := strings.TrimSuffix(strings.Repeat("((SELECT id FROM software_cpe WHERE cpe=?),?),", len(cpes)), ",") sql := fmt.Sprintf(`INSERT IGNORE INTO software_cve (cpe_id, cve) VALUES %s`, values) var args []interface{} for _, cpe := range cpes { args = append(args, cpe, cve) } - _, err := d.writer.ExecContext(ctx, sql, args...) + res, err := d.writer.ExecContext(ctx, sql, args...) if err != nil { - return ctxerr.Wrap(ctx, err, "insert software cve") + return 0, ctxerr.Wrap(ctx, err, "insert software cve") } - return nil + count, _ := res.RowsAffected() + return count, nil } func (d *Datastore) ListSoftware(ctx context.Context, opt fleet.SoftwareListOptions) ([]fleet.Software, error) { @@ -652,3 +656,37 @@ func (d *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt tim return nil } + +// HostsByCPEs returns a list of all hosts that have the software corresponding +// to at least one of the CPEs installed. It returns a minimal represention of +// matching hosts. +func (d *Datastore) HostsByCPEs(ctx context.Context, cpes []string) ([]*fleet.CPEHost, error) { + queryStmt := ` + SELECT + h.id, + h.hostname + FROM + hosts h + INNER JOIN + host_software hs + ON + h.id = hs.host_id + INNER JOIN + software_cpe scp + ON + hs.software_id = scp.software_id + WHERE + scp.cpe IN (?) + ORDER BY + h.id` + + stmt, args, err := sqlx.In(queryStmt, cpes) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "building query args") + } + var hosts []*fleet.CPEHost + if err := sqlx.SelectContext(ctx, d.reader, &hosts, stmt, args...); err != nil { + return nil, ctxerr.Wrap(ctx, err, "select hosts by cpes") + } + return hosts, nil +} diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index 1dbba9cac6..53809edc54 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -202,7 +202,14 @@ func testSoftwareInsertCVEs(t *testing.T, ds *Datastore) { require.NoError(t, ds.LoadHostSoftware(context.Background(), host)) require.NoError(t, ds.AddCPEForSoftware(context.Background(), host.Software[0], "somecpe")) - require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})) + count, err := ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"}) + require.NoError(t, err) + assert.Equal(t, int64(1), count) + + // run again for the same CPE, should not create any new row + count, err = ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"}) + require.NoError(t, err) + assert.Equal(t, int64(0), count) } func testSoftwareHostDuplicates(t *testing.T, ds *Datastore) { @@ -250,8 +257,10 @@ func testSoftwareLoadVulnerabilities(t *testing.T, ds *Datastore) { require.NoError(t, ds.AddCPEForSoftware(context.Background(), host.Software[0], "somecpe")) require.NoError(t, ds.AddCPEForSoftware(context.Background(), host.Software[1], "someothercpewithoutvulns")) - require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})) - require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-321-321-321", []string{"somecpe"})) + _, err := ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"}) + require.NoError(t, err) + _, err = ds.InsertCVEForCPE(context.Background(), "cve-321-321-321", []string{"somecpe"}) + require.NoError(t, err) require.NoError(t, ds.LoadHostSoftware(context.Background(), host)) @@ -387,8 +396,10 @@ func testSoftwareList(t *testing.T, ds *Datastore) { }) require.NoError(t, ds.AddCPEForSoftware(context.Background(), host1.Software[0], "somecpe")) require.NoError(t, ds.AddCPEForSoftware(context.Background(), host1.Software[1], "someothercpewithoutvulns")) - require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-321-432-543", []string{"somecpe"})) - require.NoError(t, ds.InsertCVEForCPE(context.Background(), "cve-333-444-555", []string{"somecpe"})) + _, err := ds.InsertCVEForCPE(context.Background(), "cve-321-432-543", []string{"somecpe"}) + require.NoError(t, err) + _, err = ds.InsertCVEForCPE(context.Background(), "cve-333-444-555", []string{"somecpe"}) + require.NoError(t, err) foo001 := fleet.Software{ Name: "foo", Version: "0.0.1", Source: "chrome_extensions", GenerateCPE: "somecpe", diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 100f5666d4..e10e90ad1f 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -328,9 +328,10 @@ type Datastore interface { AllSoftwareWithoutCPEIterator(ctx context.Context) (SoftwareIterator, error) AddCPEForSoftware(ctx context.Context, software Software, cpe string) error AllCPEs(ctx context.Context) ([]string, error) - InsertCVEForCPE(ctx context.Context, cve string, cpes []string) error + InsertCVEForCPE(ctx context.Context, cve string, cpes []string) (int64, error) SoftwareByID(ctx context.Context, id uint) (*Software, error) CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error + HostsByCPEs(ctx context.Context, cpes []string) ([]*CPEHost, error) /////////////////////////////////////////////////////////////////////////////// // ActivitiesStore diff --git a/server/fleet/hosts.go b/server/fleet/hosts.go index 295b6788ce..8d6d122d55 100644 --- a/server/fleet/hosts.go +++ b/server/fleet/hosts.go @@ -286,3 +286,10 @@ type AggregatedMacadminsData struct { MunkiVersions []AggregatedMunkiVersion `json:"munki_versions"` MDMStatus AggregatedMDMStatus `json:"mobile_device_management_enrollment_status"` } + +// CPEHost is a minimal host representation returned when querying hosts by +// CPE. +type CPEHost struct { + ID uint `json:"id" db:"id"` + Hostname string `json:"hostname" db:"hostname"` +} diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index f4afb730fa..ff75de668f 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -268,12 +268,14 @@ type AddCPEForSoftwareFunc func(ctx context.Context, software fleet.Software, cp type AllCPEsFunc func(ctx context.Context) ([]string, error) -type InsertCVEForCPEFunc func(ctx context.Context, cve string, cpes []string) error +type InsertCVEForCPEFunc func(ctx context.Context, cve string, cpes []string) (int64, error) type SoftwareByIDFunc func(ctx context.Context, id uint) (*fleet.Software, error) type CalculateHostsPerSoftwareFunc func(ctx context.Context, updatedAt time.Time) error +type HostsByCPEsFunc func(ctx context.Context, cpes []string) ([]*fleet.CPEHost, error) + type NewActivityFunc func(ctx context.Context, user *fleet.User, activityType string, details *map[string]interface{}) error type ListActivitiesFunc func(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Activity, error) @@ -762,6 +764,9 @@ type DataStore struct { CalculateHostsPerSoftwareFunc CalculateHostsPerSoftwareFunc CalculateHostsPerSoftwareFuncInvoked bool + HostsByCPEsFunc HostsByCPEsFunc + HostsByCPEsFuncInvoked bool + NewActivityFunc NewActivityFunc NewActivityFuncInvoked bool @@ -1544,7 +1549,7 @@ func (s *DataStore) AllCPEs(ctx context.Context) ([]string, error) { return s.AllCPEsFunc(ctx) } -func (s *DataStore) InsertCVEForCPE(ctx context.Context, cve string, cpes []string) error { +func (s *DataStore) InsertCVEForCPE(ctx context.Context, cve string, cpes []string) (int64, error) { s.InsertCVEForCPEFuncInvoked = true return s.InsertCVEForCPEFunc(ctx, cve, cpes) } @@ -1559,6 +1564,11 @@ func (s *DataStore) CalculateHostsPerSoftware(ctx context.Context, updatedAt tim return s.CalculateHostsPerSoftwareFunc(ctx, updatedAt) } +func (s *DataStore) HostsByCPEs(ctx context.Context, cpes []string) ([]*fleet.CPEHost, error) { + s.HostsByCPEsFuncInvoked = true + return s.HostsByCPEsFunc(ctx, cpes) +} + func (s *DataStore) NewActivity(ctx context.Context, user *fleet.User, activityType string, details *map[string]interface{}) error { s.NewActivityFuncInvoked = true return s.NewActivityFunc(ctx, user, activityType, details) diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index b605a9a0ab..03a0f03e94 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -374,7 +374,8 @@ func (s *integrationTestSuite) TestVulnerableSoftware() { } require.NoError(t, s.ds.AddCPEForSoftware(context.Background(), soft1, "somecpe")) - require.NoError(t, s.ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"})) + _, err = s.ds.InsertCVEForCPE(context.Background(), "cve-123-123-132", []string{"somecpe"}) + require.NoError(t, err) resp := s.Do("GET", fmt.Sprintf("/api/v1/fleet/hosts/%d", host.ID), nil, http.StatusOK) bodyBytes, err := ioutil.ReadAll(resp.Body) diff --git a/server/vulnerabilities/cve.go b/server/vulnerabilities/cve.go index 278513b094..41f9ada672 100644 --- a/server/vulnerabilities/cve.go +++ b/server/vulnerabilities/cve.go @@ -11,7 +11,9 @@ import ( "sync" "time" + "github.com/WatchBeam/clock" "github.com/facebookincubator/nvdtools/cvefeed" + feednvd "github.com/facebookincubator/nvdtools/cvefeed/nvd" "github.com/facebookincubator/nvdtools/providers/nvd" "github.com/facebookincubator/nvdtools/wfn" "github.com/fleetdm/fleet/v4/server/config" @@ -49,60 +51,89 @@ func SyncCVEData(vulnPath string, config config.FleetConfig) error { return dfs.Do(ctx) } +const publishedDateFmt = "2006-01-02T15:04Z" // not quite RFC3339 + +var ( + rxNVDCVEArchive = regexp.MustCompile(`nvdcve.*\.gz$`) + + // max age to be considered a recent vulnerability (relative to NVD's published date) + // (a var to be able to change in tests) + recentVulnMaxAge = 2 * 24 * time.Hour + + // this allows mocking the time package for tests, by default it is equivalent + // to the time functions, e.g. theClock.Now() == time.Now(). + theClock clock.Clock = clock.C +) + +// TranslateCPEToCVE maps the CVEs found in NVD archive files in the +// vulnerabilities database folder to software CPEs in the fleet database. +// If collectRecentVulns is true, it also returns a mapping of recent CVEs +// to a list of CPEs affected by the CVE, otherwise that map is nil. func TranslateCPEToCVE( ctx context.Context, ds fleet.Datastore, vulnPath string, logger kitlog.Logger, config config.FleetConfig, -) error { + collectRecentVulns bool, +) (map[string][]string, error) { err := SyncCVEData(vulnPath, config) if err != nil { - return err + return nil, err } var files []string err = filepath.Walk(vulnPath, func(path string, info os.FileInfo, err error) error { - if match, err := regexp.MatchString("nvdcve.*\\.gz$", path); !match || err != nil { + if match := rxNVDCVEArchive.MatchString(path); !match { return nil } files = append(files, path) return nil }) if err != nil { - return err + return nil, err + } + + if len(files) == 0 { + return nil, nil } cpeList, err := ds.AllCPEs(ctx) if err != nil { - return err + return nil, err } cpes := make([]*wfn.Attributes, 0, len(cpeList)) for _, uri := range cpeList { attr, err := wfn.Parse(uri) if err != nil { - return err + return nil, err } cpes = append(cpes, attr) } if len(cpes) == 0 { - return nil + return nil, nil } + var recentVulns map[string][]string + if collectRecentVulns { + recentVulns = make(map[string][]string) + } for _, file := range files { - err := checkCVEs(ctx, ds, logger, cpes, file) + err := checkCVEs(ctx, ds, logger, cpes, file, recentVulns) if err != nil { - return err + return nil, err } } - return nil + return recentVulns, nil } -func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, cpes []*wfn.Attributes, files ...string) error { - dict, err := cvefeed.LoadJSONDictionary(files...) +func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, + cpes []*wfn.Attributes, file string, recentVulns map[string][]string) error { + + dict, err := cvefeed.LoadJSONDictionary(file) if err != nil { return err } @@ -111,9 +142,10 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, cp //cache.Idx = cvefeed.NewIndex(dict) cpeCh := make(chan *wfn.Attributes) + collectVulns := recentVulns != nil var wg sync.WaitGroup - + var mu sync.Mutex for i := 0; i < runtime.NumCPU(); i++ { wg.Add(1) goRoutineKey := i @@ -136,10 +168,12 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, cp if ml == 0 { continue } + + cveID := matches.CVE.ID() matchingCPEs := make([]string, 0, ml) for _, attr := range matches.CPEs { if attr == nil { - level.Error(logger).Log("matches nil CPE", matches.CVE.ID()) + level.Error(logger).Log("matches nil CPE", cveID) continue } cpe := attr.BindToFmtString() @@ -148,9 +182,45 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, cp } matchingCPEs = append(matchingCPEs, cpe) } - err = ds.InsertCVEForCPE(ctx, matches.CVE.ID(), matchingCPEs) + + newCount, err := ds.InsertCVEForCPE(ctx, cveID, matchingCPEs) if err != nil { level.Error(logger).Log("cpe processing", "error", "err", err) + continue // do not report a recent vuln that failed to be inserted in the DB + } + + // collect as recent vuln only if newCount > 0, otherwise we would send + // webhook requests for the same vulnerability over and over again until + // it is older than 2 days. + if collectVulns && newCount > 0 { + vuln, ok := matches.CVE.(*feednvd.Vuln) + if !ok { + level.Error(logger).Log("recent vuln", "unexpected type for Vuln interface", "cve", cveID, + "type", fmt.Sprintf("%T", matches.CVE)) + continue + } + + rawPubDate := vuln.Schema().PublishedDate + if rawPubDate == "" { + level.Error(logger).Log("recent vuln", "empty published date", "cve", cveID) + continue + } + + pubDate, err := time.Parse(publishedDateFmt, rawPubDate) + if err != nil { + level.Error(logger).Log("recent vuln", "unexpected published date format", "cve", cveID, + "published_date", rawPubDate, "err", err) + continue + } + + // the second condition should only affect tests - to ignore pubDates in the future + // when using a mocked current clock. When using the real clock, the published date + // should always be in the past. + if theClock.Since(pubDate) <= recentVulnMaxAge && theClock.Now().After(pubDate) { + mu.Lock() + recentVulns[cveID] = append(recentVulns[cveID], matchingCPEs...) + mu.Unlock() + } } } case <-ctx.Done(): @@ -170,6 +240,5 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, cp level.Debug(logger).Log("pushing cpes", "done") wg.Wait() - return nil } diff --git a/server/vulnerabilities/cve_test.go b/server/vulnerabilities/cve_test.go index 98a0042594..a48fadd26d 100644 --- a/server/vulnerabilities/cve_test.go +++ b/server/vulnerabilities/cve_test.go @@ -12,10 +12,13 @@ import ( "strings" "sync" "testing" + "time" + "github.com/WatchBeam/clock" "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/mock" kitlog "github.com/go-kit/kit/log" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -26,7 +29,7 @@ var cvetests = []struct { {"cpe:2.3:a:1password:1password:3.9.9:*:*:*:*:*:*:*", "CVE-2012-6369"}, } -func PrintMemUsage() { +func printMemUsage() { var m runtime.MemStats runtime.ReadMemStats(&m) // For info on each, see: https://golang.org/pkg/runtime/#MemStats @@ -50,6 +53,12 @@ func TestTranslateCPEToCVE(t *testing.T) { ds := new(mock.Store) ctx := context.Background() + // download the CVEs once for all sub-tests, and then disable syncing + cfg := config.FleetConfig{} + err := SyncCVEData(tempDir, cfg) + require.NoError(t, err) + cfg.Vulnerabilities.DisableDataSync = true + for _, tt := range cvetests { t.Run(tt.cpe, func(t *testing.T) { ds.AllCPEsFunc = func(ctx context.Context) ([]string, error) { @@ -59,23 +68,70 @@ func TestTranslateCPEToCVE(t *testing.T) { cveLock := &sync.Mutex{} cveToCPEs := make(map[string][]string) var cvesFound []string - ds.InsertCVEForCPEFunc = func(ctx context.Context, cve string, cpes []string) error { + ds.InsertCVEForCPEFunc = func(ctx context.Context, cve string, cpes []string) (int64, error) { cveLock.Lock() defer cveLock.Unlock() cveToCPEs[cve] = cpes cvesFound = append(cvesFound, cve) - return nil + return 0, nil } - err := TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewLogfmtLogger(os.Stdout), config.FleetConfig{}) + _, err := TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewLogfmtLogger(os.Stdout), cfg, false) require.NoError(t, err) - PrintMemUsage() + printMemUsage() require.Equal(t, []string{tt.cve}, cvesFound) require.Equal(t, []string{tt.cpe}, cveToCPEs[tt.cve]) }) } + + t.Run("recent_vulns", func(t *testing.T) { + googleChromeCPE := "cpe:2.3:a:google:chrome:-:*:*:*:*:*:*:*" + mozillaFirefoxCPE := "cpe:2.3:a:mozilla:firefox:-:*:*:*:*:*:*:*" + curlCPE := "cpe:2.3:a:haxx:curl:-:*:*:*:*:*:*:*" + + // consider recent vulnerabilities to be anything published in 2018 + theClock = clock.NewMockClock(time.Date(2019, 01, 01, 0, 0, 0, 0, time.UTC)) + oldMaxAge := recentVulnMaxAge + recentVulnMaxAge = 365 * 24 * time.Hour + defer func() { recentVulnMaxAge = oldMaxAge; theClock = clock.C }() + + ds.AllCPEsFunc = func(ctx context.Context) ([]string, error) { + return []string{googleChromeCPE, mozillaFirefoxCPE, curlCPE}, nil + } + + ds.InsertCVEForCPEFunc = func(ctx context.Context, cve string, cpes []string) (int64, error) { + return 1, nil + } + recent, err := TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewNopLogger(), cfg, true) + require.NoError(t, err) + + byCPE := make(map[string]int) + for _, cpes := range recent { + for _, cpe := range cpes { + byCPE[cpe]++ + } + } + + // even if it's somewhat far in the past, I've seen the exact numbers + // change a bit between runs with different downloads, so allow for a bit + // of wiggle room. + assert.Greater(t, byCPE[googleChromeCPE], 150, "google chrome CVEs") + assert.Greater(t, byCPE[mozillaFirefoxCPE], 280, "mozilla firefox CVEs") + assert.Greater(t, byCPE[curlCPE], 10, "curl CVEs") + + // call it again but now return 0 from this call, simulating CVE-CPE pairs + // that already existed in the DB. + ds.InsertCVEForCPEFunc = func(ctx context.Context, cve string, cpes []string) (int64, error) { + return 0, nil + } + recent, err = TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewNopLogger(), cfg, true) + require.NoError(t, err) + + // no recent vulnerability should be reported + assert.Len(t, recent, 0) + }) } func TestSyncsCVEFromURL(t *testing.T) { diff --git a/server/webhooks/vulnerabilities.go b/server/webhooks/vulnerabilities.go new file mode 100644 index 0000000000..2c66f14dfb --- /dev/null +++ b/server/webhooks/vulnerabilities.go @@ -0,0 +1,94 @@ +package webhooks + +import ( + "context" + "fmt" + "net/url" + "path" + "strconv" + "time" + + "github.com/fleetdm/fleet/v4/server" + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" + kitlog "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" +) + +// TriggerVulnerabilitiesWebhook performs the webhook requests for vulnerabilities. +func TriggerVulnerabilitiesWebhook( + ctx context.Context, + ds fleet.Datastore, + logger kitlog.Logger, + recentVulns map[string][]string, + appConfig *fleet.AppConfig, + now time.Time, +) error { + vulnConfig := appConfig.WebhookSettings.VulnerabilitiesWebhook + if !vulnConfig.Enable { + return nil + } + + level.Debug(logger).Log("enabled", "true", "recentVulns", len(recentVulns)) + + serverURL, err := url.Parse(appConfig.ServerSettings.ServerURL) + if err != nil { + return ctxerr.Wrap(ctx, err, "invalid server url") + } + + targetURL := vulnConfig.DestinationURL + batchSize := vulnConfig.HostBatchSize + + for cve, cpes := range recentVulns { + hosts, err := ds.HostsByCPEs(ctx, cpes) + if err != nil { + return ctxerr.Wrap(ctx, err, "get hosts by CPE") + } + + for len(hosts) > 0 { + limit := len(hosts) + if batchSize > 0 && len(hosts) > batchSize { + limit = batchSize + } + if err := sendVulnerabilityHostBatch(ctx, targetURL, cve, serverURL, hosts[:limit], now); err != nil { + return ctxerr.Wrap(ctx, err, "send vulnerability host batch") + } + hosts = hosts[limit:] + } + } + + return nil +} + +type vulnHostPayload struct { + ID uint `json:"id"` + Hostname string `json:"hostname"` + URL string `json:"url"` +} + +func sendVulnerabilityHostBatch(ctx context.Context, targetURL, cve string, hostBaseURL *url.URL, hosts []*fleet.CPEHost, now time.Time) error { + shortHosts := make([]*vulnHostPayload, len(hosts)) + for i, h := range hosts { + hostURL := *hostBaseURL + hostURL.Path = path.Join(hostURL.Path, "hosts", strconv.Itoa(int(h.ID))) + shortHosts[i] = &vulnHostPayload{ + ID: h.ID, + Hostname: h.Hostname, + URL: hostURL.String(), + } + } + + payload := map[string]interface{}{ + "timestamp": now, + "vulnerability": map[string]interface{}{ + "cve": cve, + "details_link": fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cve), + "hosts_affected": shortHosts, + }, + } + + if err := server.PostJSONWithTimeout(ctx, targetURL, &payload); err != nil { + return ctxerr.Wrapf(ctx, err, "posting to %s", targetURL) + } + return nil +} diff --git a/server/webhooks/vulnerabilities_test.go b/server/webhooks/vulnerabilities_test.go new file mode 100644 index 0000000000..06199be8e3 --- /dev/null +++ b/server/webhooks/vulnerabilities_test.go @@ -0,0 +1,151 @@ +package webhooks + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/mock" + kitlog "github.com/go-kit/kit/log" + "github.com/stretchr/testify/require" + "github.com/tj/assert" +) + +func TestTriggerVulnerabilitiesWebhook(t *testing.T) { + ctx := context.Background() + ds := new(mock.Store) + logger := kitlog.NewNopLogger() + + appCfg := &fleet.AppConfig{ + WebhookSettings: fleet.WebhookSettings{ + VulnerabilitiesWebhook: fleet.VulnerabilitiesWebhookSettings{ + Enable: true, + HostBatchSize: 2, + }, + }, + ServerSettings: fleet.ServerSettings{ + ServerURL: "https://fleet.example.com", + }, + } + + recentVulns := map[string][]string{ + "CVE-2012-1234": {"cpe1", "cpe2"}, + } + + t.Run("disabled", func(t *testing.T) { + appCfg := *appCfg + appCfg.WebhookSettings.VulnerabilitiesWebhook.Enable = false + err := TriggerVulnerabilitiesWebhook(ctx, ds, logger, recentVulns, &appCfg, time.Now()) + require.NoError(t, err) + }) + + t.Run("invalid server url", func(t *testing.T) { + appCfg := *appCfg + appCfg.ServerSettings.ServerURL = ":nope:" + err := TriggerVulnerabilitiesWebhook(ctx, ds, logger, recentVulns, &appCfg, time.Now()) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid server") + }) + + t.Run("empty recent vulns", func(t *testing.T) { + err := TriggerVulnerabilitiesWebhook(ctx, ds, logger, nil, appCfg, time.Now()) + require.NoError(t, err) + }) + + t.Run("trigger requests", func(t *testing.T) { + now := time.Now() + + hosts := []*fleet.CPEHost{ + {ID: 1, Hostname: "h1"}, + {ID: 2, Hostname: "h2"}, + {ID: 3, Hostname: "h3"}, + {ID: 4, Hostname: "h4"}, + } + jsonH1 := fmt.Sprintf(`{"id":1,"hostname":"h1","url":"%s/hosts/1"}`, appCfg.ServerSettings.ServerURL) + jsonH2 := fmt.Sprintf(`{"id":2,"hostname":"h2","url":"%s/hosts/2"}`, appCfg.ServerSettings.ServerURL) + jsonH3 := fmt.Sprintf(`{"id":3,"hostname":"h3","url":"%s/hosts/3"}`, appCfg.ServerSettings.ServerURL) + jsonH4 := fmt.Sprintf(`{"id":4,"hostname":"h4","url":"%s/hosts/4"}`, appCfg.ServerSettings.ServerURL) + + cves := []string{ + "CVE-2012-1234", + "CVE-2012-4567", + } + jsonCVE1 := fmt.Sprintf(`{"timestamp":"%s","vulnerability":{"cve":%q,"details_link":"https://nvd.nist.gov/vuln/detail/%[2]s","hosts_affected":`, + now.Format(time.RFC3339Nano), cves[0]) + jsonCVE2 := fmt.Sprintf(`{"timestamp":"%s","vulnerability":{"cve":%q,"details_link":"https://nvd.nist.gov/vuln/detail/%[2]s","hosts_affected":`, + now.Format(time.RFC3339Nano), cves[1]) + + cases := []struct { + name string + vulns map[string][]string + hosts []*fleet.CPEHost + want string + }{ + { + "1 vuln, 1 host", + map[string][]string{cves[0]: {"cpe1"}}, + hosts[:1], + fmt.Sprintf("%s[%s]}}", jsonCVE1, jsonH1), + }, + { + "1 vuln, 2 hosts", + map[string][]string{cves[0]: {"cpe1"}}, + hosts[:2], + fmt.Sprintf("%s[%s,%s]}}", jsonCVE1, jsonH1, jsonH2), + }, + { + "1 vuln, 3 hosts", + map[string][]string{cves[0]: {"cpe1"}}, + hosts[:3], + fmt.Sprintf("%s[%s,%s]}}\n%s[%s]}}", jsonCVE1, jsonH1, jsonH2, jsonCVE1, jsonH3), // 2 requests, batch of 2 max + }, + { + "1 vuln, 4 hosts", + map[string][]string{cves[0]: {"cpe1"}}, + hosts[:4], + fmt.Sprintf("%s[%s,%s]}}\n%s[%s,%s]}}", jsonCVE1, jsonH1, jsonH2, jsonCVE1, jsonH3, jsonH4), // 2 requests, batch of 2 max + }, + { + "2 vulns, 1 host each", + map[string][]string{cves[0]: {"cpe1"}, cves[1]: {"cpe2"}}, + hosts[:1], + fmt.Sprintf("%s[%s]}}\n%s[%s]}}", jsonCVE1, jsonH1, jsonCVE2, jsonH1), + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + var requests []string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, err := ioutil.ReadAll(r.Body) + assert.NoError(t, err) + requests = append(requests, string(b)) + w.Write(nil) + })) + defer srv.Close() + + ds.HostsByCPEsFunc = func(ctx context.Context, cpes []string) ([]*fleet.CPEHost, error) { + return c.hosts, nil + } + + appCfg := *appCfg + appCfg.WebhookSettings.VulnerabilitiesWebhook.DestinationURL = srv.URL + err := TriggerVulnerabilitiesWebhook(ctx, ds, logger, c.vulns, &appCfg, now) + require.NoError(t, err) + + assert.True(t, ds.HostsByCPEsFuncInvoked) + ds.HostsByCPEsFuncInvoked = false + + want := strings.Split(c.want, "\n") + assert.ElementsMatch(t, want, requests) + }) + } + }) +}