From 4c2ddba2e475f8534e62a8d28cd72fb2dae9d9c4 Mon Sep 17 00:00:00 2001 From: Juan Fernandez Date: Mon, 3 Apr 2023 13:45:18 -0400 Subject: [PATCH] Clean out-of-date NVD results. (#10514) Keep the vulnerabilities detected via NVD and stored in the DB in sync. with the results from the NVD vulnerability process. --- changes/clean_out_old_nvd_results | 2 + cmd/fleet/cron.go | 2 +- cmd/fleet/serve_test.go | 30 +- ...30134823_AddSoftwareCPEUniqueConstraint.go | 61 +++ ...823_AddSoftwareCPEUniqueConstraint_test.go | 41 ++ server/datastore/mysql/schema.sql | 6 +- server/datastore/mysql/software.go | 160 +++--- server/datastore/mysql/software_test.go | 495 ++++++++++++------ server/fleet/datastore.go | 23 +- server/fleet/software.go | 10 + server/fleet/software_test.go | 56 ++ server/mock/datastore_mock.go | 60 ++- server/service/integration_core_test.go | 41 +- server/service/integration_enterprise_test.go | 12 +- server/vulnerabilities/macoffice/analyzer.go | 19 +- .../macoffice/analyzer_test.go | 8 +- .../macoffice/integration_analyzer_test.go | 12 +- server/vulnerabilities/nvd/cpe.go | 74 ++- server/vulnerabilities/nvd/cpe_test.go | 83 ++- server/vulnerabilities/nvd/cve.go | 14 +- server/vulnerabilities/nvd/cve_test.go | 36 +- server/vulnerabilities/oval/analyzer.go | 17 +- server/vulnerabilities/oval/analyzer_test.go | 6 +- 23 files changed, 913 insertions(+), 355 deletions(-) create mode 100644 changes/clean_out_old_nvd_results create mode 100644 server/datastore/mysql/migrations/tables/20230330134823_AddSoftwareCPEUniqueConstraint.go create mode 100644 server/datastore/mysql/migrations/tables/20230330134823_AddSoftwareCPEUniqueConstraint_test.go create mode 100644 server/fleet/software_test.go diff --git a/changes/clean_out_old_nvd_results b/changes/clean_out_old_nvd_results new file mode 100644 index 0000000000..617c5b6a61 --- /dev/null +++ b/changes/clean_out_old_nvd_results @@ -0,0 +1,2 @@ +- If a new CPE translation rule is pushed, the data in the database should reflect that. +- If a false positive is patched, the data in the database should reflect that. diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index 04379ab75b..d86ed52563 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -368,7 +368,7 @@ func checkNVDVulnerabilities( return nil } - vulns, err := nvd.TranslateCPEToCVE(ctx, ds, vulnPath, logger, collectVulns) + vulns, err := nvd.TranslateCPEToCVE(ctx, ds, vulnPath, logger, collectVulns, config.Periodicity) if err != nil { errHandler(ctx, logger, "analyzing vulnerable software: CPE->CVE", err) return nil diff --git a/cmd/fleet/serve_test.go b/cmd/fleet/serve_test.go index 8243f99e0e..c1fdbbfab3 100644 --- a/cmd/fleet/serve_test.go +++ b/cmd/fleet/serve_test.go @@ -271,7 +271,7 @@ func TestCronVulnerabilitiesCreatesDatabasesPath(t *testing.T) { ds.InsertCVEMetaFunc = func(ctx context.Context, x []fleet.CVEMeta) error { return nil } - ds.AllSoftwareWithoutCPEIteratorFunc = func(ctx context.Context, excludedPlatforms []string) (fleet.SoftwareIterator, error) { + ds.AllSoftwareIteratorFunc = func(ctx context.Context, query fleet.SoftwareIterQueryOptions) (fleet.SoftwareIterator, error) { // we should not get this far before we see the directory being created return nil, errors.New("shouldn't happen") } @@ -388,7 +388,7 @@ func TestScanVulnerabilities(t *testing.T) { ds.InsertCVEMetaFunc = func(ctx context.Context, x []fleet.CVEMeta) error { return nil } - ds.AllSoftwareWithoutCPEIteratorFunc = func(ctx context.Context, excludedPlatforms []string) (fleet.SoftwareIterator, error) { + ds.AllSoftwareIteratorFunc = func(ctx context.Context, query fleet.SoftwareIterQueryOptions) (fleet.SoftwareIterator, error) { iterator := &softwareIterator{ softwares: []*fleet.Software{ { @@ -411,10 +411,16 @@ func TestScanVulnerabilities(t *testing.T) { }, }, nil } - ds.InsertSoftwareVulnerabilitiesFunc = func(ctx context.Context, vulns []fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (int64, error) { - return 1, nil + ds.InsertSoftwareVulnerabilityFunc = func(ctx context.Context, vuln fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (bool, error) { + return true, nil } - ds.AddCPEForSoftwareFunc = func(ctx context.Context, software fleet.Software, cpe string) error { + ds.UpsertSoftwareCPEsFunc = func(ctx context.Context, cpes []fleet.SoftwareCPE) (int64, error) { + return int64(0), nil + } + ds.DeleteSoftwareCPEsFunc = func(ctx context.Context, cpes []fleet.SoftwareCPE) (int64, error) { + return int64(0), nil + } + ds.DeleteOutOfDateVulnerabilitiesFunc = func(ctx context.Context, source fleet.VulnerabilitySource, duration time.Duration) error { return nil } ds.OSVersionsFunc = func(ctx context.Context, teamID *uint, platform *string, name *string, version *string) (*fleet.OSVersions, error) { @@ -480,20 +486,6 @@ func TestScanVulnerabilities(t *testing.T) { }, }, nil } - ds.ListSoftwareBySourceIterFunc = func(ctx context.Context, sources []string) (fleet.SoftwareIterator, error) { - iterator := &softwareIterator{ - softwares: []*fleet.Software{ - { - ID: 1, - Name: "Twisted", - Version: "22.2.0", - BundleIdentifier: "", - Source: "python_packages", - }, - }, - } - return iterator, nil - } vulnPath := t.TempDir() diff --git a/server/datastore/mysql/migrations/tables/20230330134823_AddSoftwareCPEUniqueConstraint.go b/server/datastore/mysql/migrations/tables/20230330134823_AddSoftwareCPEUniqueConstraint.go new file mode 100644 index 0000000000..93b5a2684a --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20230330134823_AddSoftwareCPEUniqueConstraint.go @@ -0,0 +1,61 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20230330134823, Down_20230330134823) +} + +// Since we will be adding a uniqueness constrain on (software_id) on the software_cpe table - we need to remove any +// possible duplicates. +func _20230329161600_remove_duplicates(tx *sql.Tx) error { + const deleteStmt = ` +DELETE sc +FROM software_cpe sc + INNER JOIN ( + SELECT + software_id, + MAX(id) as max_id + FROM software_cpe + GROUP BY software_id + HAVING COUNT(*) > 1 + ) sc2 ON sc2.software_id = sc.software_id + WHERE sc.id < sc2.max_id; +` + + if _, err := tx.Exec(deleteStmt); err != nil { + return errors.Wrap(err, "removing duplicated rows") + } + + return nil +} + +func _20230329161600_add_unq_constraint(tx *sql.Tx) error { + _, err := tx.Exec(` + ALTER TABLE software_cpe ADD CONSTRAINT unq_software_id UNIQUE (software_id), ALGORITHM=INPLACE, LOCK=NONE; +`) + if err != nil { + return errors.Wrapf(err, "adding unique constraint to software_id on software_cpe") + } + return nil +} + +func Up_20230330134823(tx *sql.Tx) error { + if err := _20230329161600_remove_duplicates(tx); err != nil { + return err + } + + if err := _20230329161600_add_unq_constraint(tx); err != nil { + return err + } + + return nil +} + +func Down_20230330134823(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20230330134823_AddSoftwareCPEUniqueConstraint_test.go b/server/datastore/mysql/migrations/tables/20230330134823_AddSoftwareCPEUniqueConstraint_test.go new file mode 100644 index 0000000000..03727fb3fa --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20230330134823_AddSoftwareCPEUniqueConstraint_test.go @@ -0,0 +1,41 @@ +package tables + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUp_20230330134823(t *testing.T) { + db := applyUpToPrev(t) + + _, err := db.Exec(`INSERT INTO software (id, name, version, source, bundle_identifier, vendor, arch) + VALUES (1, 'zchunk-libs', '1.2.1', 'rpm_packages', '', 'Fedora Project','x86_64');`) + require.NoError(t, err) + + _, err = db.Exec(`INSERT INTO software_cpe (software_id, cpe, created_at, updated_at) + VALUES (1, 'some_cpe', '2022-06-19 18:04:02', '2022-07-04 14:33:04');`) + require.NoError(t, err) + + _, err = db.Exec(`INSERT INTO software_cpe (software_id, cpe, created_at, updated_at) + VALUES (1, 'some_cpe', '2022-06-19 18:04:02', '2022-07-04 14:33:04');`) + require.NoError(t, err) + + applyNext(t, db) + + var n uint + + // Test we removed dup + err = db.QueryRow(`SELECT COUNT(1) FROM software_cpe`).Scan(&n) + require.NoError(t, err) + require.Equal(t, uint(1), n) + + // Test unique constraint + _, err = db.Exec(`INSERT IGNORE INTO software_cpe (software_id, cpe, created_at, updated_at) + VALUES (1, 'some_cpe', '2022-06-19 18:04:02', '2022-07-04 14:33:04');`) + require.NoError(t, err) + + err = db.QueryRow(`SELECT COUNT(1) FROM software_cpe`).Scan(&n) + require.NoError(t, err) + require.Equal(t, uint(1), n) +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index cdb2a89510..2d155c7a20 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -551,9 +551,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=177 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=178 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `mobile_device_management_solutions` ( @@ -1020,7 +1020,7 @@ CREATE TABLE `software_cpe` ( `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `cpe` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, PRIMARY KEY (`id`), - KEY `fk_software_cpe_software_id` (`software_id`), + UNIQUE KEY `unq_software_id` (`software_id`), KEY `software_cpe_cpe_idx` (`cpe`), CONSTRAINT `software_cpe_ibfk_1` FOREIGN KEY (`software_id`) REFERENCES `software` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 5f6d662aec..65b634c053 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -663,56 +663,48 @@ func (si *softwareIterator) Next() bool { return si.rows.Next() } -func (ds *Datastore) ListSoftwareBySourceIter( +// AllSoftwareIterator Returns an iterator for the 'software' table, filtering out +// software entries based on the 'query' param. The rows.Close call is done by the caller once +// iteration using the returned fleet.SoftwareIterator is done. +func (ds *Datastore) AllSoftwareIterator( ctx context.Context, - sources []string, + query fleet.SoftwareIterQueryOptions, ) (fleet.SoftwareIterator, error) { - if len(sources) == 0 { - return nil, errors.New("please provide at least one source") + if !query.IsValid() { + return nil, fmt.Errorf("invalid query params %+v", query) } var err error var args []interface{} stmt := `SELECT - s.id, - s.name, - s.version, - s.bundle_identifier, - s.release, - s.vendor, - s.arch, - s.source - FROM software s - WHERE source IN (?)` + s.* , + COALESCE(sc.cpe, '') AS generated_cpe + FROM software s + LEFT JOIN software_cpe sc ON (s.id=sc.software_id)` - stmt, args, err = sqlx.In(stmt, sources) + var conditionals []string + arg := map[string]interface{}{} - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "error while trying to bind sources") + if len(query.ExcludedSources) != 0 { + conditionals = append(conditionals, "s.source NOT IN (:excluded_sources)") + arg["excluded_sources"] = query.ExcludedSources } - rows, err := ds.reader.QueryxContext(ctx, stmt, args...) //nolint:sqlclosecheck - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "error executing SQL statement") + if len(query.IncludedSources) != 0 { + conditionals = append(conditionals, "s.source IN (:included_sources)") + arg["included_sources"] = query.IncludedSources } - return &softwareIterator{rows: rows}, nil -} -// AllSoftwareWithoutCPEIterator Returns an iterator for the 'software' table, filtering out -// software entries with CPEs and from the sources included in the 'excludedSources' param. -func (ds *Datastore) AllSoftwareWithoutCPEIterator(ctx context.Context, excludedSources []string) (fleet.SoftwareIterator, error) { - var err error - var args []interface{} - - stmt := `SELECT s.* FROM software s LEFT JOIN software_cpe sc ON (s.id=sc.software_id) WHERE sc.id IS NULL` - // The rows.Close call is done by the caller once iteration using the - // returned fleet.SoftwareIterator is done. - if excludedSources != nil { - stmt += ` AND s.source NOT IN (?)` - stmt, args, err = sqlx.In(stmt, excludedSources) + if len(conditionals) != 0 { + cond := strings.Join(conditionals, " AND ") + stmt, args, err = sqlx.Named(stmt+" WHERE "+cond, arg) if err != nil { - return nil, ctxerr.Wrap(ctx, err, "loads cpes") + return nil, ctxerr.Wrap(ctx, err, "error binding named arguments on software iterator") + } + stmt, args, err = sqlx.In(stmt, args...) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "error building 'In' query part on software iterator") } } @@ -723,19 +715,56 @@ func (ds *Datastore) AllSoftwareWithoutCPEIterator(ctx context.Context, excluded return &softwareIterator{rows: rows}, nil } -func (ds *Datastore) AddCPEForSoftware(ctx context.Context, software fleet.Software, cpe string) error { - _, err := addCPEForSoftwareDB(ctx, ds.writer, software, cpe) - return err +func (ds *Datastore) UpsertSoftwareCPEs(ctx context.Context, cpes []fleet.SoftwareCPE) (int64, error) { + var args []interface{} + + if len(cpes) == 0 { + return 0, nil + } + + values := strings.TrimSuffix(strings.Repeat("(?,?),", len(cpes)), ",") + sql := fmt.Sprintf( + `INSERT INTO software_cpe (software_id, cpe) VALUES %s ON DUPLICATE KEY UPDATE cpe = VALUES(cpe)`, + values, + ) + + for _, cpe := range cpes { + args = append(args, cpe.SoftwareID, cpe.CPE) + } + res, err := ds.writer.ExecContext(ctx, sql, args...) + if err != nil { + return 0, ctxerr.Wrap(ctx, err, "insert software cpes") + } + count, _ := res.RowsAffected() + + return count, nil } -func addCPEForSoftwareDB(ctx context.Context, exec sqlx.ExecerContext, software fleet.Software, cpe string) (uint, error) { - sql := `INSERT INTO software_cpe (software_id, cpe) VALUES (?, ?)` - res, err := exec.ExecContext(ctx, sql, software.ID, cpe) - if err != nil { - return 0, ctxerr.Wrap(ctx, err, "insert software cpe") +func (ds *Datastore) DeleteSoftwareCPEs(ctx context.Context, cpes []fleet.SoftwareCPE) (int64, error) { + if len(cpes) == 0 { + return 0, nil } - id, _ := res.LastInsertId() // cannot fail with the mysql driver - return uint(id), nil + + stmt := `DELETE FROM software_cpe WHERE (software_id) IN (?)` + + softwareIDs := make([]uint, 0, len(cpes)) + for _, cpe := range cpes { + softwareIDs = append(softwareIDs, cpe.SoftwareID) + } + + query, args, err := sqlx.In(stmt, softwareIDs) + if err != nil { + return 0, ctxerr.Wrap(ctx, err, "error building 'In' query part when deleting software CPEs") + } + + res, err := ds.writer.ExecContext(ctx, query, args...) + if err != nil { + return 0, ctxerr.Wrapf(ctx, err, "deleting cpes software") + } + + count, _ := res.RowsAffected() + + return count, nil } func (ds *Datastore) ListSoftwareCPEs(ctx context.Context) ([]fleet.SoftwareCPE, error) { @@ -781,6 +810,19 @@ func (ds *Datastore) DeleteSoftwareVulnerabilities(ctx context.Context, vulnerab return nil } +func (ds *Datastore) DeleteOutOfDateVulnerabilities(ctx context.Context, source fleet.VulnerabilitySource, duration time.Duration) error { + sql := `DELETE FROM software_cve WHERE source = ? AND updated_at < ?` + + var args []interface{} + cutPoint := time.Now().UTC().Add(-1 * duration) + args = append(args, source, cutPoint) + + if _, err := ds.writer.ExecContext(ctx, sql, args...); err != nil { + return ctxerr.Wrap(ctx, err, "deleting out of date vulnerabilities") + } + return nil +} + func (ds *Datastore) SoftwareByID(ctx context.Context, id uint, includeCVEScores bool) (*fleet.Software, error) { q := dialect.From(goqu.I("software").As("s")). Select( @@ -1097,30 +1139,26 @@ ON DUPLICATE KEY UPDATE return nil } -func (ds *Datastore) InsertSoftwareVulnerabilities( +func (ds *Datastore) InsertSoftwareVulnerability( ctx context.Context, - vulns []fleet.SoftwareVulnerability, + vuln fleet.SoftwareVulnerability, source fleet.VulnerabilitySource, -) (int64, error) { +) (bool, error) { + if vuln.CVE == "" { + return false, nil + } + var args []interface{} - if len(vulns) == 0 { - return 0, nil - } + stmt := `INSERT INTO software_cve (cve, source, software_id) VALUES (?,?,?) ON DUPLICATE KEY UPDATE updated_at=?` + args = append(args, vuln.CVE, source, vuln.SoftwareID, time.Now().UTC()) - values := strings.TrimSuffix(strings.Repeat("(?,?,?),", len(vulns)), ",") - sql := fmt.Sprintf(`INSERT IGNORE INTO software_cve (cve, source, software_id) VALUES %s`, values) - - for _, v := range vulns { - args = append(args, v.CVE, source, v.SoftwareID) - } - res, err := ds.writer.ExecContext(ctx, sql, args...) + res, err := ds.writer.ExecContext(ctx, stmt, args...) if err != nil { - return 0, ctxerr.Wrap(ctx, err, "insert software vulnerabilities") + return false, ctxerr.Wrap(ctx, err, "insert software vulnerability") } - count, _ := res.RowsAffected() - return count, nil + return insertOnDuplicateDidInsert(res), nil } func (ds *Datastore) ListSoftwareVulnerabilitiesByHostIDsSource( diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index d7a7c1ec2e..e0480dc752 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -15,6 +15,7 @@ import ( "github.com/fleetdm/fleet/v4/server/vulnerabilities/oval" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/exp/slices" ) func TestSoftware(t *testing.T) { @@ -37,12 +38,15 @@ func TestSoftware(t *testing.T) { {"HostsByCVE", testHostsByCVE}, {"HostsBySoftwareIDs", testHostsBySoftwareIDs}, {"UpdateHostSoftware", testUpdateHostSoftware}, - {"ListSoftwareBySourceIter", testListSoftwareBySourceIter}, {"ListSoftwareByHostIDShort", testListSoftwareByHostIDShort}, {"ListSoftwareVulnerabilitiesByHostIDsSource", testListSoftwareVulnerabilitiesByHostIDsSource}, - {"InsertSoftwareVulnerabilities", testInsertSoftwareVulnerabilities}, + {"InsertSoftwareVulnerability", testInsertSoftwareVulnerability}, {"ListCVEs", testListCVEs}, {"ListSoftwareForVulnDetection", testListSoftwareForVulnDetection}, + {"AllSoftwareIterator", testAllSoftwareIterator}, + {"UpsertSoftwareCPEs", testUpsertSoftwareCPEs}, + {"DeleteOutOfDateVulnerabilities", testDeleteOutOfDateVulnerabilities}, + {"DeleteSoftwareCPEs", testDeleteSoftwareCPEs}, {"SoftwareByIDNoDuplicatedVulns", testSoftwareByIDNoDuplicatedVulns}, {"SoftwareByIDIncludesCVEPublishedDate", testSoftwareByIDIncludesCVEPublishedDate}, } @@ -149,20 +153,18 @@ func testSoftwareCPE(t *testing.T, ds *Datastore) { err = ds.UpdateHostSoftware(context.Background(), host1.ID, software2) require.NoError(t, err) - iterator, err := ds.AllSoftwareWithoutCPEIterator(context.Background(), oval.SupportedSoftwareSources) - defer iterator.Close() + q := fleet.SoftwareIterQueryOptions{ExcludedSources: oval.SupportedSoftwareSources} + iterator, err := ds.AllSoftwareIterator(context.Background(), q) require.NoError(t, err) + defer iterator.Close() loops := 0 - id := uint(0) for iterator.Next() { software, err := iterator.Value() require.NoError(t, err) require.NoError(t, iterator.Err()) require.NotEmpty(t, software.ID) - id = software.ID - require.NotEmpty(t, software.Name) require.NotEmpty(t, software.Version) require.NotEmpty(t, software.Source) @@ -177,34 +179,6 @@ func testSoftwareCPE(t *testing.T, ds *Datastore) { } assert.Equal(t, len(software1), loops) require.NoError(t, iterator.Close()) - - err = ds.AddCPEForSoftware(context.Background(), fleet.Software{ID: id}, "some:cpe") - require.NoError(t, err) - - iterator, err = ds.AllSoftwareWithoutCPEIterator(context.Background(), oval.SupportedSoftwareSources) - defer iterator.Close() - require.NoError(t, err) - - loops = 0 - for iterator.Next() { - software, err := iterator.Value() - require.NoError(t, err) - require.NoError(t, iterator.Err()) - - require.NotEmpty(t, software.ID) - require.NotEqual(t, id, software.ID) - - require.NotEmpty(t, software.Name) - require.NotEmpty(t, software.Version) - require.NotEmpty(t, software.Source) - - if loops > 1 { - t.Error("Looping through more software than we have") - } - loops++ - } - assert.Equal(t, len(software1)-1, loops) - require.NoError(t, iterator.Close()) } func testSoftwareHostDuplicates(t *testing.T, ds *Datastore) { @@ -252,17 +226,22 @@ func testSoftwareLoadVulnerabilities(t *testing.T, ds *Datastore) { require.NoError(t, ds.UpdateHostSoftware(context.Background(), host.ID, software)) require.NoError(t, ds.LoadHostSoftware(context.Background(), host, false)) - require.NoError(t, ds.AddCPEForSoftware(context.Background(), host.Software[0], "somecpe")) - require.NoError(t, ds.AddCPEForSoftware(context.Background(), host.Software[1], "someothercpewithoutvulns")) + cpes := []fleet.SoftwareCPE{ + {SoftwareID: host.Software[0].ID, CPE: "somecpe"}, + {SoftwareID: host.Software[1].ID, CPE: "someothercpewithoutvulns"}, + } + _, err := ds.UpsertSoftwareCPEs(context.Background(), cpes) + require.NoError(t, err) require.NoError(t, ds.LoadHostSoftware(context.Background(), host, false)) vulns := []fleet.SoftwareVulnerability{ {SoftwareID: host.Software[0].ID, CVE: "CVE-2022-0001"}, {SoftwareID: host.Software[0].ID, CVE: "CVE-2022-0002"}, } - _, err := ds.InsertSoftwareVulnerabilities(context.Background(), vulns, fleet.NVDSource) - require.NoError(t, err) - + for _, v := range vulns { + _, err = ds.InsertSoftwareVulnerability(context.Background(), v, fleet.NVDSource) + require.NoError(t, err) + } require.NoError(t, ds.LoadHostSoftware(context.Background(), host, false)) softByID, err := ds.SoftwareByID(context.Background(), host.HostSoftware.Software[0].ID, false) @@ -306,13 +285,16 @@ func testListSoftwareCPEs(t *testing.T, ds *Datastore) { require.NoError(t, ds.UpdateHostSoftware(ctx, ubuntu.ID, software[2:])) require.NoError(t, ds.LoadHostSoftware(ctx, ubuntu, false)) - require.NoError(t, ds.AddCPEForSoftware(ctx, debian.Software[0], "cpe1")) - require.NoError(t, ds.AddCPEForSoftware(ctx, debian.Software[1], "cpe2")) + cpes := []fleet.SoftwareCPE{ + {SoftwareID: debian.Software[0].ID, CPE: "cpe1"}, + {SoftwareID: debian.Software[1].ID, CPE: "cpe2"}, + {SoftwareID: ubuntu.Software[0].ID, CPE: "cpe3"}, + {SoftwareID: ubuntu.Software[1].ID, CPE: "cpe4"}, + } + _, err := ds.UpsertSoftwareCPEs(ctx, cpes) + require.NoError(t, err) - require.NoError(t, ds.AddCPEForSoftware(ctx, ubuntu.Software[0], "cpe3")) - require.NoError(t, ds.AddCPEForSoftware(ctx, ubuntu.Software[1], "cpe4")) - - cpes, err := ds.ListSoftwareCPEs(ctx) + cpes, err = ds.ListSoftwareCPEs(ctx) expected := []string{ "cpe1", "cpe2", "cpe3", "cpe4", } @@ -427,9 +409,11 @@ func testSoftwareLoadSupportsTonsOfCVEs(t *testing.T, ds *Datastore) { sort.Slice(host.Software, func(i, j int) bool { return host.Software[i].Name < host.Software[j].Name }) - require.NoError(t, ds.AddCPEForSoftware(context.Background(), host.Software[1], "someothercpewithoutvulns")) - - _, err := addCPEForSoftwareDB(context.Background(), ds.writer, host.Software[0], "somecpe") + cpes := []fleet.SoftwareCPE{ + {SoftwareID: host.Software[1].ID, CPE: "someothercpewithoutvulns"}, + {SoftwareID: host.Software[0].ID, CPE: "somecpe"}, + } + _, err := ds.UpsertSoftwareCPEs(context.Background(), cpes) require.NoError(t, err) var cveMeta []fleet.CVEMeta @@ -499,9 +483,13 @@ func testSoftwareList(t *testing.T, ds *Datastore) { return host1.Software[i].Name+host1.Software[i].Version < host1.Software[j].Name+host1.Software[j].Version }) - require.NoError(t, ds.AddCPEForSoftware(context.Background(), host1.Software[0], "somecpe")) - require.NoError(t, ds.AddCPEForSoftware(context.Background(), host1.Software[1], "someothercpewithoutvulns")) - require.NoError(t, ds.AddCPEForSoftware(context.Background(), host3.Software[0], "somecpe2")) + cpes := []fleet.SoftwareCPE{ + {SoftwareID: host1.Software[0].ID, CPE: "somecpe"}, + {SoftwareID: host1.Software[1].ID, CPE: "someothercpewithoutvulns"}, + {SoftwareID: host3.Software[0].ID, CPE: "somecpe2"}, + } + _, err := ds.UpsertSoftwareCPEs(context.Background(), cpes) + require.NoError(t, err) require.NoError(t, ds.LoadHostSoftware(context.Background(), host1, false)) require.NoError(t, ds.LoadHostSoftware(context.Background(), host2, false)) @@ -516,8 +504,10 @@ func testSoftwareList(t *testing.T, ds *Datastore) { {SoftwareID: host3.Software[0].ID, CVE: "CVE-2022-0003"}, } - _, err := ds.InsertSoftwareVulnerabilities(context.Background(), vulns, fleet.NVDSource) - require.NoError(t, err) + for _, v := range vulns { + _, err = ds.InsertSoftwareVulnerability(context.Background(), v, fleet.NVDSource) + require.NoError(t, err) + } now := time.Now().UTC().Truncate(time.Second) cveMeta := []fleet.CVEMeta{ @@ -1126,12 +1116,15 @@ func insertVulnSoftwareForTest(t *testing.T, ds *Datastore) { return host2.Software[i].Name+host2.Software[i].Version < host2.Software[j].Name+host2.Software[j].Version }) - require.NoError(t, ds.AddCPEForSoftware(context.Background(), host1.Software[0], "cpe_foo_chrome_3")) - require.NoError(t, ds.AddCPEForSoftware(context.Background(), host1.Software[1], "cpe_foo_rpm")) - - require.NoError(t, ds.AddCPEForSoftware(context.Background(), host2.Software[0], "cpe_bar_rpm")) - require.NoError(t, ds.AddCPEForSoftware(context.Background(), host2.Software[1], "cpe_foo_chrome_2")) - require.NoError(t, ds.AddCPEForSoftware(context.Background(), host2.Software[2], "cpe_foo_chrome_3")) + cpes := []fleet.SoftwareCPE{ + {SoftwareID: host1.Software[0].ID, CPE: "cpe_foo_chrome_3"}, + {SoftwareID: host1.Software[1].ID, CPE: "cpe_foo_rpm"}, + {SoftwareID: host2.Software[0].ID, CPE: "cpe_bar_rpm"}, + {SoftwareID: host2.Software[1].ID, CPE: "cpe_foo_chrome_2"}, + {SoftwareID: host2.Software[2].ID, CPE: "cpe_foo_chrome_3"}, + } + _, err := ds.UpsertSoftwareCPEs(context.Background(), cpes) + require.NoError(t, err) require.NoError(t, ds.LoadHostSoftware(context.Background(), host1, false)) require.NoError(t, ds.LoadHostSoftware(context.Background(), host2, false)) @@ -1143,31 +1136,31 @@ func insertVulnSoftwareForTest(t *testing.T, ds *Datastore) { }) chrome3 := host2.Software[2] - n, err := ds.InsertSoftwareVulnerabilities(context.Background(), []fleet.SoftwareVulnerability{ - { - SoftwareID: chrome3.ID, - CVE: "CVE-2022-0001", - }, + inserted, err := ds.InsertSoftwareVulnerability(context.Background(), fleet.SoftwareVulnerability{ + SoftwareID: chrome3.ID, + CVE: "CVE-2022-0001", }, fleet.NVDSource) require.NoError(t, err) - require.Equal(t, 1, int(n)) + require.True(t, inserted) barRpm := host2.Software[0] - n, err = ds.InsertSoftwareVulnerabilities(context.Background(), - []fleet.SoftwareVulnerability{ - { - SoftwareID: barRpm.ID, - CVE: "CVE-2022-0002", - }, - { - SoftwareID: barRpm.ID, - CVE: "CVE-2022-0003", - }, - }, fleet.NVDSource) + vulns := []fleet.SoftwareVulnerability{ + { + SoftwareID: barRpm.ID, + CVE: "CVE-2022-0002", + }, + { + SoftwareID: barRpm.ID, + CVE: "CVE-2022-0003", + }, + } - require.NoError(t, err) - require.Equal(t, 2, int(n)) + for _, v := range vulns { + inserted, err := ds.InsertSoftwareVulnerability(context.Background(), v, fleet.NVDSource) + require.NoError(t, err) + require.True(t, inserted) + } require.NoError(t, ds.SyncHostsSoftware(context.Background(), time.Now())) } @@ -1414,39 +1407,6 @@ func testUpdateHostSoftware(t *testing.T, ds *Datastore) { validateSoftware(tup{"bar", lastYear}, tup{"baz", future}, tup{"qux", future}) } -func testListSoftwareBySourceIter(t *testing.T, ds *Datastore) { - host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) - - software := []fleet.Software{ - {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, - {Name: "foo", Version: "0.0.3", Source: "chrome_extensions"}, - {Name: "foo", Version: "v0.0.2", Source: "apps"}, - {Name: "foo", Version: "0.0.3", Source: "apps"}, - {Name: "bar", Version: "0.0.3", Source: "deb_packages"}, - } - - require.NoError(t, ds.UpdateHostSoftware(context.Background(), host.ID, software)) - - expected := []fleet.Software{ - {Name: "foo", Version: "v0.0.2", Source: "apps"}, - {Name: "foo", Version: "0.0.3", Source: "apps"}, - } - - var actual []fleet.Software - - iter, err := ds.ListSoftwareBySourceIter(context.Background(), []string{"apps"}) - require.NoError(t, err) - defer iter.Close() - - for iter.Next() { - software, err := iter.Value() - require.NoError(t, err) - actual = append(actual, *software) - } - - test.ElementsMatchSkipID(t, expected, actual) -} - func testListSoftwareByHostIDShort(t *testing.T, ds *Datastore) { host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) @@ -1493,9 +1453,14 @@ func testListSoftwareVulnerabilitiesByHostIDsSource(t *testing.T, ds *Datastore) require.NoError(t, ds.UpdateHostSoftware(ctx, host.ID, software)) require.NoError(t, ds.LoadHostSoftware(ctx, host, false)) - require.NoError(t, ds.AddCPEForSoftware(ctx, host.Software[0], "foo_cpe")) - require.NoError(t, ds.AddCPEForSoftware(ctx, host.Software[1], "bar_cpe")) - require.NoError(t, ds.AddCPEForSoftware(ctx, host.Software[2], "blah_cpe")) + cpes := []fleet.SoftwareCPE{ + {SoftwareID: host.Software[0].ID, CPE: "foo_cpe"}, + {SoftwareID: host.Software[1].ID, CPE: "bar_cpe"}, + {SoftwareID: host.Software[2].ID, CPE: "blah_cpe"}, + } + _, err := ds.UpsertSoftwareCPEs(ctx, cpes) + require.NoError(t, err) + require.NoError(t, ds.LoadHostSoftware(ctx, host, false)) cveMap := map[int]string{ @@ -1503,21 +1468,18 @@ func testListSoftwareVulnerabilitiesByHostIDsSource(t *testing.T, ds *Datastore) 1: "cve-456", } - var vulns []fleet.SoftwareVulnerability for i, s := range host.Software { cve, ok := cveMap[i] if ok { - vulns = append(vulns, fleet.SoftwareVulnerability{ + inserted, err := ds.InsertSoftwareVulnerability(ctx, fleet.SoftwareVulnerability{ SoftwareID: s.ID, CVE: cve, - }) + }, fleet.NVDSource) + require.NoError(t, err) + require.True(t, inserted) } } - n, err := ds.InsertSoftwareVulnerabilities(ctx, vulns, fleet.NVDSource) - require.NoError(t, err) - require.Equal(t, int64(2), n) - result, err := ds.ListSoftwareVulnerabilitiesByHostIDsSource(ctx, []uint{host.ID}, fleet.NVDSource) require.NoError(t, err) @@ -1534,13 +1496,13 @@ func testListSoftwareVulnerabilitiesByHostIDsSource(t *testing.T, ds *Datastore) } } -func testInsertSoftwareVulnerabilities(t *testing.T, ds *Datastore) { +func testInsertSoftwareVulnerability(t *testing.T, ds *Datastore) { ctx := context.Background() t.Run("no vulnerabilities to insert", func(t *testing.T) { - r, err := ds.InsertSoftwareVulnerabilities(ctx, nil, fleet.UbuntuOVALSource) + inserted, err := ds.InsertSoftwareVulnerability(ctx, fleet.SoftwareVulnerability{}, fleet.UbuntuOVALSource) require.NoError(t, err) - require.Zero(t, r) + require.False(t, inserted) }) t.Run("duplicated vulnerabilities", func(t *testing.T) { @@ -1551,21 +1513,23 @@ func testInsertSoftwareVulnerabilities(t *testing.T, ds *Datastore) { require.NoError(t, ds.UpdateHostSoftware(ctx, host.ID, []fleet.Software{software})) require.NoError(t, ds.LoadHostSoftware(ctx, host, false)) - require.NoError(t, ds.AddCPEForSoftware(ctx, host.Software[0], "foo_cpe_1")) - - var vulns []fleet.SoftwareVulnerability - for _, s := range host.Software { - vulns = append(vulns, fleet.SoftwareVulnerability{ - SoftwareID: s.ID, CVE: "cve-1", - }) - vulns = append(vulns, fleet.SoftwareVulnerability{ - SoftwareID: s.ID, CVE: "cve-1", - }) + cpes := []fleet.SoftwareCPE{ + {SoftwareID: host.Software[0].ID, CPE: "foo_cpe_1"}, } - - n, err := ds.InsertSoftwareVulnerabilities(ctx, vulns, fleet.UbuntuOVALSource) + _, err := ds.UpsertSoftwareCPEs(ctx, cpes) require.NoError(t, err) - require.Equal(t, int64(1), n) + + inserted, err := ds.InsertSoftwareVulnerability(ctx, fleet.SoftwareVulnerability{ + SoftwareID: host.Software[0].ID, CVE: "cve-1", + }, fleet.UbuntuOVALSource) + require.NoError(t, err) + require.True(t, inserted) + + inserted, err = ds.InsertSoftwareVulnerability(ctx, fleet.SoftwareVulnerability{ + SoftwareID: host.Software[0].ID, CVE: "cve-1", + }, fleet.UbuntuOVALSource) + require.NoError(t, err) + require.False(t, inserted) storedVulns, err := ds.ListSoftwareVulnerabilitiesByHostIDsSource(ctx, []uint{host.ID}, fleet.UbuntuOVALSource) require.NoError(t, err) @@ -1585,7 +1549,11 @@ func testInsertSoftwareVulnerabilities(t *testing.T, ds *Datastore) { require.NoError(t, ds.UpdateHostSoftware(ctx, host.ID, []fleet.Software{software})) require.NoError(t, ds.LoadHostSoftware(ctx, host, false)) - require.NoError(t, ds.AddCPEForSoftware(ctx, host.Software[0], "foo_cpe_2")) + cpes := []fleet.SoftwareCPE{ + {SoftwareID: host.Software[0].ID, CPE: "foo_cpe_2"}, + } + _, err := ds.UpsertSoftwareCPEs(ctx, cpes) + require.NoError(t, err) var vulns []fleet.SoftwareVulnerability for _, s := range host.Software { @@ -1595,13 +1563,13 @@ func testInsertSoftwareVulnerabilities(t *testing.T, ds *Datastore) { }) } - n, err := ds.InsertSoftwareVulnerabilities(ctx, vulns, fleet.UbuntuOVALSource) + inserted, err := ds.InsertSoftwareVulnerability(ctx, vulns[0], fleet.UbuntuOVALSource) require.NoError(t, err) - require.Equal(t, 1, int(n)) + require.True(t, inserted) - n, err = ds.InsertSoftwareVulnerabilities(ctx, vulns, fleet.UbuntuOVALSource) + inserted, err = ds.InsertSoftwareVulnerability(ctx, vulns[0], fleet.UbuntuOVALSource) require.NoError(t, err) - require.Equal(t, 0, int(n)) + require.False(t, inserted) storedVulns, err := ds.ListSoftwareVulnerabilitiesByHostIDsSource(ctx, []uint{host.ID}, fleet.UbuntuOVALSource) require.NoError(t, err) @@ -1661,7 +1629,8 @@ func testListSoftwareForVulnDetection(t *testing.T, ds *Datastore) { } require.NoError(t, ds.UpdateHostSoftware(ctx, host.ID, software)) require.NoError(t, ds.LoadHostSoftware(ctx, host, false)) - require.NoError(t, ds.AddCPEForSoftware(ctx, host.Software[0], "cpe1")) + _, err := ds.UpsertSoftwareCPEs(ctx, []fleet.SoftwareCPE{{SoftwareID: host.Software[0].ID, CPE: "cpe1"}}) + require.NoError(t, err) // Load software again so that CPE data is included. require.NoError(t, ds.LoadHostSoftware(ctx, host, false)) @@ -1711,14 +1680,14 @@ func testSoftwareByIDNoDuplicatedVulns(t *testing.T, ds *Datastore) { // Add one vulnerability to each software var vulns []fleet.SoftwareVulnerability for i, s := range hostA.Software { - vulns = append(vulns, fleet.SoftwareVulnerability{ + inserted, err := ds.InsertSoftwareVulnerability(ctx, fleet.SoftwareVulnerability{ SoftwareID: s.ID, CVE: fmt.Sprintf("cve-%d", i), - }) + }, fleet.UbuntuOVALSource) + require.NoError(t, err) + require.True(t, inserted) + vulns = append(vulns) } - n, err := ds.InsertSoftwareVulnerabilities(ctx, vulns, fleet.UbuntuOVALSource) - require.NoError(t, err) - require.Equal(t, 4, int(n)) for _, s := range hostA.Software { result, err := ds.SoftwareByID(ctx, s.ID, true) @@ -1761,7 +1730,6 @@ func testSoftwareByIDIncludesCVEPublishedDate(t *testing.T, ds *Datastore) { require.NoError(t, ds.LoadHostSoftware(ctx, host, false)) // Add vulnerabilities and CVEMeta - var vulns []fleet.SoftwareVulnerability var meta []fleet.CVEMeta for _, tC := range testCases { idx := -1 @@ -1774,10 +1742,12 @@ func testSoftwareByIDIncludesCVEPublishedDate(t *testing.T, ds *Datastore) { require.NotEqual(t, -1, idx, "software not found") if tC.hasVuln { - vulns = append(vulns, fleet.SoftwareVulnerability{ + inserted, err := ds.InsertSoftwareVulnerability(ctx, fleet.SoftwareVulnerability{ SoftwareID: host.Software[idx].ID, CVE: fmt.Sprintf("cve-%s", tC.name), - }) + }, fleet.UbuntuOVALSource) + require.NoError(t, err) + require.True(t, inserted) } if tC.hasMeta { @@ -1795,9 +1765,6 @@ func testSoftwareByIDIncludesCVEPublishedDate(t *testing.T, ds *Datastore) { }) } } - n, err := ds.InsertSoftwareVulnerabilities(ctx, vulns, fleet.UbuntuOVALSource) - require.NoError(t, err) - require.Equal(t, 3, int(n)) require.NoError(t, ds.InsertCVEMeta(ctx, meta)) for _, tC := range testCases { @@ -1855,3 +1822,221 @@ func testSoftwareByIDIncludesCVEPublishedDate(t *testing.T, ds *Datastore) { } }) } + +func testAllSoftwareIterator(t *testing.T, ds *Datastore) { + host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) + + software := []fleet.Software{ + {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, + {Name: "foo", Version: "0.0.3", Source: "chrome_extensions"}, + {Name: "foo", Version: "v0.0.2", Source: "apps"}, + {Name: "foo", Version: "0.0.3", Source: "apps"}, + {Name: "bar", Version: "0.0.3", Source: "deb_packages"}, + } + require.NoError(t, ds.UpdateHostSoftware(context.Background(), host.ID, software)) + require.NoError(t, ds.LoadHostSoftware(context.Background(), host, false)) + + foo_ce_v1 := slices.IndexFunc(host.Software, func(c fleet.Software) bool { + return c.Name == "foo" && c.Version == "0.0.1" && c.Source == "chrome_extensions" + }) + foo_app_v2 := slices.IndexFunc(host.Software, func(c fleet.Software) bool { return c.Name == "foo" && c.Version == "v0.0.2" && c.Source == "apps" }) + bar_v3 := slices.IndexFunc(host.Software, func(c fleet.Software) bool { + return c.Name == "bar" && c.Version == "0.0.3" && c.Source == "deb_packages" + }) + + cpes := []fleet.SoftwareCPE{ + {SoftwareID: host.Software[foo_ce_v1].ID, CPE: "cpe:foo_ce_v1"}, + {SoftwareID: host.Software[foo_app_v2].ID, CPE: "cpe:foo_app_v2"}, + {SoftwareID: host.Software[bar_v3].ID, CPE: "cpe:bar_v3"}, + } + _, err := ds.UpsertSoftwareCPEs(context.Background(), cpes) + require.NoError(t, err) + + testCases := []struct { + q fleet.SoftwareIterQueryOptions + expected []fleet.Software + }{ + { + expected: []fleet.Software{ + {Name: "foo", Version: "v0.0.2", Source: "apps", GenerateCPE: "cpe:foo_app_v2"}, + {Name: "foo", Version: "0.0.3", Source: "apps"}, + }, + q: fleet.SoftwareIterQueryOptions{IncludedSources: []string{"apps"}}, + }, + { + expected: []fleet.Software{ + {Name: "foo", Version: "0.0.1", Source: "chrome_extensions", GenerateCPE: "cpe:foo_ce_v1"}, + {Name: "foo", Version: "0.0.3", Source: "chrome_extensions"}, + {Name: "bar", Version: "0.0.3", Source: "deb_packages", GenerateCPE: "cpe:bar_v3"}, + }, + q: fleet.SoftwareIterQueryOptions{ExcludedSources: []string{"apps"}}, + }, + { + expected: []fleet.Software{ + {Name: "foo", Version: "v0.0.2", Source: "apps", GenerateCPE: "cpe:foo_app_v2"}, + {Name: "foo", Version: "0.0.3", Source: "apps"}, + }, + q: fleet.SoftwareIterQueryOptions{IncludedSources: []string{"apps"}}, + }, + { + expected: []fleet.Software{ + {Name: "foo", Version: "0.0.1", Source: "chrome_extensions", GenerateCPE: "cpe:foo_ce_v1"}, + {Name: "foo", Version: "v0.0.2", Source: "apps", GenerateCPE: "cpe:foo_app_v2"}, + {Name: "foo", Version: "0.0.3", Source: "apps"}, + {Name: "foo", Version: "0.0.3", Source: "chrome_extensions"}, + {Name: "bar", Version: "0.0.3", Source: "deb_packages", GenerateCPE: "cpe:bar_v3"}, + }, + q: fleet.SoftwareIterQueryOptions{}, + }, + } + + for _, tC := range testCases { + var actual []fleet.Software + + iter, err := ds.AllSoftwareIterator(context.Background(), tC.q) + require.NoError(t, err) + for iter.Next() { + software, err := iter.Value() + require.NoError(t, err) + actual = append(actual, *software) + } + iter.Close() + test.ElementsMatchSkipID(t, tC.expected, actual) + } +} + +func testUpsertSoftwareCPEs(t *testing.T, ds *Datastore) { + ctx := context.Background() + host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) + + software := []fleet.Software{ + {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, + } + require.NoError(t, ds.UpdateHostSoftware(ctx, host.ID, software)) + require.NoError(t, ds.LoadHostSoftware(ctx, host, false)) + + cpes := []fleet.SoftwareCPE{ + {SoftwareID: host.Software[0].ID, CPE: "cpe:foo_ce_v1"}, + {SoftwareID: host.Software[0].ID, CPE: "cpe:foo_ce_v2"}, + } + _, err := ds.UpsertSoftwareCPEs(ctx, cpes) + require.NoError(t, err) + + cpes, err = ds.ListSoftwareCPEs(ctx) + require.NoError(t, err) + require.Equal(t, len(cpes), 1) + require.Equal(t, cpes[0].CPE, "cpe:foo_ce_v2") + + cpes = []fleet.SoftwareCPE{ + {SoftwareID: host.Software[0].ID, CPE: "cpe:foo_ce_v3"}, + } + _, err = ds.UpsertSoftwareCPEs(ctx, cpes) + require.NoError(t, err) + + cpes = []fleet.SoftwareCPE{ + {SoftwareID: host.Software[0].ID, CPE: "cpe:foo_ce_v4"}, + } + _, err = ds.UpsertSoftwareCPEs(ctx, cpes) + require.NoError(t, err) + + cpes, err = ds.ListSoftwareCPEs(ctx) + require.NoError(t, err) + require.Equal(t, len(cpes), 1) + require.Equal(t, cpes[0].CPE, "cpe:foo_ce_v4") +} + +func testDeleteOutOfDateVulnerabilities(t *testing.T, ds *Datastore) { + ctx := context.Background() + host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) + + software := []fleet.Software{ + {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, + } + require.NoError(t, ds.UpdateHostSoftware(ctx, host.ID, software)) + require.NoError(t, ds.LoadHostSoftware(ctx, host, false)) + + vulns := []fleet.SoftwareVulnerability{ + { + SoftwareID: host.Software[0].ID, + CVE: "CVE-2023-001", + }, + { + SoftwareID: host.Software[0].ID, + CVE: "CVE-2023-002", + }, + } + + inserted, err := ds.InsertSoftwareVulnerability(ctx, vulns[0], fleet.NVDSource) + require.NoError(t, err) + require.True(t, inserted) + + inserted, err = ds.InsertSoftwareVulnerability(ctx, vulns[1], fleet.NVDSource) + require.NoError(t, err) + require.True(t, inserted) + + _, err = ds.writer.ExecContext(ctx, "UPDATE software_cve SET updated_at = '2020-10-10 12:00:00'") + require.NoError(t, err) + + // This should update the 'updated_at' timestamp. + inserted, err = ds.InsertSoftwareVulnerability(ctx, vulns[0], fleet.NVDSource) + require.NoError(t, err) + require.False(t, inserted) + + err = ds.DeleteOutOfDateVulnerabilities(ctx, fleet.NVDSource, 2*time.Hour) + require.NoError(t, err) + + storedSoftware, err := ds.SoftwareByID(ctx, host.Software[0].ID, false) + require.NoError(t, err) + require.Equal(t, 1, len(storedSoftware.Vulnerabilities)) + require.Equal(t, "CVE-2023-001", storedSoftware.Vulnerabilities[0].CVE) +} + +func testDeleteSoftwareCPEs(t *testing.T, ds *Datastore) { + ctx := context.Background() + host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) + + software := []fleet.Software{ + {Name: "foo", Version: "0.0.1", Source: "chrome_extensions"}, + {Name: "bar", Version: "0.0.1", Source: "chrome_extensions"}, + } + require.NoError(t, ds.UpdateHostSoftware(ctx, host.ID, software)) + require.NoError(t, ds.LoadHostSoftware(ctx, host, false)) + + cpes := []fleet.SoftwareCPE{ + { + SoftwareID: host.Software[0].ID, + CPE: "CPE-001", + }, + { + SoftwareID: host.Software[1].ID, + CPE: "CPE-002", + }, + } + _, err := ds.UpsertSoftwareCPEs(ctx, cpes) + require.NoError(t, err) + + t.Run("nothing to delete", func(t *testing.T) { + affected, err := ds.DeleteSoftwareCPEs(ctx, nil) + require.NoError(t, err) + require.Zero(t, affected) + }) + + t.Run("with invalid software id", func(t *testing.T) { + toDelete := []fleet.SoftwareCPE{cpes[0], { + SoftwareID: host.Software[1].ID + 1234, + CPE: "CPE-002", + }} + + affected, err := ds.DeleteSoftwareCPEs(ctx, toDelete) + require.NoError(t, err) + require.Equal(t, int64(1), affected) + + storedCPEs, err := ds.ListSoftwareCPEs(ctx) + require.NoError(t, err) + test.ElementsMatchSkipID(t, cpes[1:], storedCPEs) + + storedSoftware, err := ds.SoftwareByID(ctx, cpes[0].SoftwareID, false) + require.NoError(t, err) + require.Empty(t, storedSoftware.GenerateCPE) + }) +} diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index e51a344e8c..da5c9e74a0 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -398,16 +398,18 @@ type Datastore interface { ListSoftwareVulnerabilitiesByHostIDsSource(ctx context.Context, hostIDs []uint, source VulnerabilitySource) (map[uint][]SoftwareVulnerability, error) LoadHostSoftware(ctx context.Context, host *Host, includeCVEScores bool) error - // ListSoftwareBySourceIter returns an iterator for consuming all software rows filtered by - // their 'source'. - ListSoftwareBySourceIter(ctx context.Context, sources []string) (SoftwareIterator, error) - - AllSoftwareWithoutCPEIterator(ctx context.Context, excludedPlatforms []string) (SoftwareIterator, error) - AddCPEForSoftware(ctx context.Context, software Software, cpe string) error + AllSoftwareIterator(ctx context.Context, query SoftwareIterQueryOptions) (SoftwareIterator, error) + // UpsertSoftwareCPEs either inserts new 'software_cpe' entries, or if a now with the same CPE + // already exists, performs an update operation. Returns the number of rows affected. + UpsertSoftwareCPEs(ctx context.Context, cpes []SoftwareCPE) (int64, error) + // DeleteSoftwareCPEs removes entries from 'software_cpe' by matching the software_id in the + // provided cpes. Returns the number of rows affected. + DeleteSoftwareCPEs(ctx context.Context, cpes []SoftwareCPE) (int64, error) ListSoftwareCPEs(ctx context.Context) ([]SoftwareCPE, error) - // InsertSoftwareVulnerabilities inserts the given vulnerabilities in the datastore, returns the number - // of rows inserted. If a vulnerability already exists in the datastore, then it will be ignored. - InsertSoftwareVulnerabilities(ctx context.Context, vulns []SoftwareVulnerability, source VulnerabilitySource) (int64, error) + // InsertSoftwareVulnerability will either insert a new vulnerability in the datastore (in which + // case it will return true) or if a matching record already exists it will update its + // updated_at timestamp (in which case it will return false). + InsertSoftwareVulnerability(ctx context.Context, vuln SoftwareVulnerability, source VulnerabilitySource) (bool, error) SoftwareByID(ctx context.Context, id uint, includeCVEScores bool) (*Software, error) // ListSoftwareByHostIDShort lists software by host ID, but does not include CPEs or vulnerabilites. // It is meant to be used when only minimal software fields are required eg when updating host software. @@ -497,6 +499,9 @@ type Datastore interface { CountSoftware(ctx context.Context, opt SoftwareListOptions) (int, error) // DeleteVulnerabilities deletes the given list of vulnerabilities identified by CPE+CVE. DeleteSoftwareVulnerabilities(ctx context.Context, vulnerabilities []SoftwareVulnerability) error + // DeleteOutOfDateVulnerabilities deletes 'software_cve' entries from the provided source where + // the updated_at timestamp is older than the provided duration + DeleteOutOfDateVulnerabilities(ctx context.Context, source VulnerabilitySource, duration time.Duration) error /////////////////////////////////////////////////////////////////////////////// // Team Policies diff --git a/server/fleet/software.go b/server/fleet/software.go index 2a6c3651a6..ce4021fb16 100644 --- a/server/fleet/software.go +++ b/server/fleet/software.go @@ -100,3 +100,13 @@ type SoftwareListOptions struct { // a count of hosts > 0. WithHostCounts bool } + +type SoftwareIterQueryOptions struct { + ExcludedSources []string // what sources to exclude + IncludedSources []string // what sources to include +} + +// IsValid checks that either ExcludedSources or IncludedSources is specified but not both +func (siqo SoftwareIterQueryOptions) IsValid() bool { + return !(len(siqo.IncludedSources) != 0 && len(siqo.ExcludedSources) != 0) +} diff --git a/server/fleet/software_test.go b/server/fleet/software_test.go new file mode 100644 index 0000000000..6fa5986d0a --- /dev/null +++ b/server/fleet/software_test.go @@ -0,0 +1,56 @@ +package fleet + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSoftwareIterQueryOptionsIsValid(t *testing.T) { + testCases := []struct { + excluded []string + included []string + isNotValid bool + }{ + { + excluded: nil, + included: nil, + }, + { + excluded: []string{"a", "b"}, + included: nil, + }, + { + excluded: nil, + included: []string{"a", "b"}, + }, + { + excluded: []string{"a", "b"}, + included: []string{"a"}, + isNotValid: true, + }, + { + excluded: []string{"a"}, + included: []string{"a", "b"}, + isNotValid: true, + }, + { + excluded: []string{"c"}, + included: []string{"a", "b"}, + isNotValid: true, + }, + } + + for _, tC := range testCases { + sut := SoftwareIterQueryOptions{ + ExcludedSources: tC.excluded, + IncludedSources: tC.included, + } + + if tC.isNotValid { + require.False(t, sut.IsValid()) + } else { + require.True(t, sut.IsValid()) + } + } +} diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 58459b7198..10dc55e3da 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -312,15 +312,15 @@ type ListSoftwareVulnerabilitiesByHostIDsSourceFunc func(ctx context.Context, ho type LoadHostSoftwareFunc func(ctx context.Context, host *fleet.Host, includeCVEScores bool) error -type ListSoftwareBySourceIterFunc func(ctx context.Context, sources []string) (fleet.SoftwareIterator, error) +type AllSoftwareIteratorFunc func(ctx context.Context, query fleet.SoftwareIterQueryOptions) (fleet.SoftwareIterator, error) -type AllSoftwareWithoutCPEIteratorFunc func(ctx context.Context, excludedPlatforms []string) (fleet.SoftwareIterator, error) +type UpsertSoftwareCPEsFunc func(ctx context.Context, cpes []fleet.SoftwareCPE) (int64, error) -type AddCPEForSoftwareFunc func(ctx context.Context, software fleet.Software, cpe string) error +type DeleteSoftwareCPEsFunc func(ctx context.Context, cpes []fleet.SoftwareCPE) (int64, error) type ListSoftwareCPEsFunc func(ctx context.Context) ([]fleet.SoftwareCPE, error) -type InsertSoftwareVulnerabilitiesFunc func(ctx context.Context, vulns []fleet.SoftwareVulnerability, source fleet.VulnerabilitySource) (int64, error) +type InsertSoftwareVulnerabilityFunc func(ctx context.Context, vuln fleet.SoftwareVulnerability, source fleet.VulnerabilitySource) (bool, error) type SoftwareByIDFunc func(ctx context.Context, id uint, includeCVEScores bool) (*fleet.Software, error) @@ -388,6 +388,8 @@ type CountSoftwareFunc func(ctx context.Context, opt fleet.SoftwareListOptions) type DeleteSoftwareVulnerabilitiesFunc func(ctx context.Context, vulnerabilities []fleet.SoftwareVulnerability) error +type DeleteOutOfDateVulnerabilitiesFunc func(ctx context.Context, source fleet.VulnerabilitySource, duration time.Duration) error + type NewTeamPolicyFunc func(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) type ListTeamPoliciesFunc func(ctx context.Context, teamID uint) (teamPolicies []*fleet.Policy, inheritedPolicies []*fleet.Policy, err error) @@ -1023,20 +1025,20 @@ type DataStore struct { LoadHostSoftwareFunc LoadHostSoftwareFunc LoadHostSoftwareFuncInvoked bool - ListSoftwareBySourceIterFunc ListSoftwareBySourceIterFunc - ListSoftwareBySourceIterFuncInvoked bool + AllSoftwareIteratorFunc AllSoftwareIteratorFunc + AllSoftwareIteratorFuncInvoked bool - AllSoftwareWithoutCPEIteratorFunc AllSoftwareWithoutCPEIteratorFunc - AllSoftwareWithoutCPEIteratorFuncInvoked bool + UpsertSoftwareCPEsFunc UpsertSoftwareCPEsFunc + UpsertSoftwareCPEsFuncInvoked bool - AddCPEForSoftwareFunc AddCPEForSoftwareFunc - AddCPEForSoftwareFuncInvoked bool + DeleteSoftwareCPEsFunc DeleteSoftwareCPEsFunc + DeleteSoftwareCPEsFuncInvoked bool ListSoftwareCPEsFunc ListSoftwareCPEsFunc ListSoftwareCPEsFuncInvoked bool - InsertSoftwareVulnerabilitiesFunc InsertSoftwareVulnerabilitiesFunc - InsertSoftwareVulnerabilitiesFuncInvoked bool + InsertSoftwareVulnerabilityFunc InsertSoftwareVulnerabilityFunc + InsertSoftwareVulnerabilityFuncInvoked bool SoftwareByIDFunc SoftwareByIDFunc SoftwareByIDFuncInvoked bool @@ -1137,6 +1139,9 @@ type DataStore struct { DeleteSoftwareVulnerabilitiesFunc DeleteSoftwareVulnerabilitiesFunc DeleteSoftwareVulnerabilitiesFuncInvoked bool + DeleteOutOfDateVulnerabilitiesFunc DeleteOutOfDateVulnerabilitiesFunc + DeleteOutOfDateVulnerabilitiesFuncInvoked bool + NewTeamPolicyFunc NewTeamPolicyFunc NewTeamPolicyFuncInvoked bool @@ -2461,25 +2466,25 @@ func (s *DataStore) LoadHostSoftware(ctx context.Context, host *fleet.Host, incl return s.LoadHostSoftwareFunc(ctx, host, includeCVEScores) } -func (s *DataStore) ListSoftwareBySourceIter(ctx context.Context, sources []string) (fleet.SoftwareIterator, error) { +func (s *DataStore) AllSoftwareIterator(ctx context.Context, query fleet.SoftwareIterQueryOptions) (fleet.SoftwareIterator, error) { s.mu.Lock() - s.ListSoftwareBySourceIterFuncInvoked = true + s.AllSoftwareIteratorFuncInvoked = true s.mu.Unlock() - return s.ListSoftwareBySourceIterFunc(ctx, sources) + return s.AllSoftwareIteratorFunc(ctx, query) } -func (s *DataStore) AllSoftwareWithoutCPEIterator(ctx context.Context, excludedPlatforms []string) (fleet.SoftwareIterator, error) { +func (s *DataStore) UpsertSoftwareCPEs(ctx context.Context, cpes []fleet.SoftwareCPE) (int64, error) { s.mu.Lock() - s.AllSoftwareWithoutCPEIteratorFuncInvoked = true + s.UpsertSoftwareCPEsFuncInvoked = true s.mu.Unlock() - return s.AllSoftwareWithoutCPEIteratorFunc(ctx, excludedPlatforms) + return s.UpsertSoftwareCPEsFunc(ctx, cpes) } -func (s *DataStore) AddCPEForSoftware(ctx context.Context, software fleet.Software, cpe string) error { +func (s *DataStore) DeleteSoftwareCPEs(ctx context.Context, cpes []fleet.SoftwareCPE) (int64, error) { s.mu.Lock() - s.AddCPEForSoftwareFuncInvoked = true + s.DeleteSoftwareCPEsFuncInvoked = true s.mu.Unlock() - return s.AddCPEForSoftwareFunc(ctx, software, cpe) + return s.DeleteSoftwareCPEsFunc(ctx, cpes) } func (s *DataStore) ListSoftwareCPEs(ctx context.Context) ([]fleet.SoftwareCPE, error) { @@ -2489,11 +2494,11 @@ func (s *DataStore) ListSoftwareCPEs(ctx context.Context) ([]fleet.SoftwareCPE, return s.ListSoftwareCPEsFunc(ctx) } -func (s *DataStore) InsertSoftwareVulnerabilities(ctx context.Context, vulns []fleet.SoftwareVulnerability, source fleet.VulnerabilitySource) (int64, error) { +func (s *DataStore) InsertSoftwareVulnerability(ctx context.Context, vuln fleet.SoftwareVulnerability, source fleet.VulnerabilitySource) (bool, error) { s.mu.Lock() - s.InsertSoftwareVulnerabilitiesFuncInvoked = true + s.InsertSoftwareVulnerabilityFuncInvoked = true s.mu.Unlock() - return s.InsertSoftwareVulnerabilitiesFunc(ctx, vulns, source) + return s.InsertSoftwareVulnerabilityFunc(ctx, vuln, source) } func (s *DataStore) SoftwareByID(ctx context.Context, id uint, includeCVEScores bool) (*fleet.Software, error) { @@ -2727,6 +2732,13 @@ func (s *DataStore) DeleteSoftwareVulnerabilities(ctx context.Context, vulnerabi return s.DeleteSoftwareVulnerabilitiesFunc(ctx, vulnerabilities) } +func (s *DataStore) DeleteOutOfDateVulnerabilities(ctx context.Context, source fleet.VulnerabilitySource, duration time.Duration) error { + s.mu.Lock() + s.DeleteOutOfDateVulnerabilitiesFuncInvoked = true + s.mu.Unlock() + return s.DeleteOutOfDateVulnerabilitiesFunc(ctx, source, duration) +} + func (s *DataStore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) { s.mu.Lock() s.NewTeamPolicyFuncInvoked = true diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index 61c1ab0c6c..f0ce568dbc 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -632,7 +632,9 @@ func (s *integrationTestSuite) TestVulnerableSoftware() { soft1 = host.Software[1] } - require.NoError(t, s.ds.AddCPEForSoftware(context.Background(), soft1, "somecpe")) + cpes := []fleet.SoftwareCPE{{SoftwareID: soft1.ID, CPE: "somecpe"}} + _, err = s.ds.UpsertSoftwareCPEs(context.Background(), cpes) + require.NoError(t, err) // Reload software so that 'GeneratedCPEID is set. require.NoError(t, s.ds.LoadHostSoftware(context.Background(), host, false)) @@ -641,16 +643,14 @@ func (s *integrationTestSuite) TestVulnerableSoftware() { soft1 = host.Software[1] } - n, err := s.ds.InsertSoftwareVulnerabilities( - context.Background(), []fleet.SoftwareVulnerability{ - { - SoftwareID: soft1.ID, - CVE: "cve-123-123-132", - }, + inserted, err := s.ds.InsertSoftwareVulnerability( + context.Background(), fleet.SoftwareVulnerability{ + SoftwareID: soft1.ID, + CVE: "cve-123-123-132", }, fleet.NVDSource, ) require.NoError(t, err) - require.Equal(t, 1, int(n)) + require.True(t, inserted) resp := s.Do("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d", host.ID), nil, http.StatusOK) bodyBytes, err := ioutil.ReadAll(resp.Body) @@ -4984,25 +4984,26 @@ func (s *integrationTestSuite) TestPaginateListSoftware() { } } + var cpes []fleet.SoftwareCPE for i, sw := range sws { - cpe := "somecpe" + strconv.Itoa(i) - require.NoError(t, s.ds.AddCPEForSoftware(context.Background(), sw, cpe)) + cpes = append(cpes, fleet.SoftwareCPE{SoftwareID: sw.ID, CPE: "somecpe" + strconv.Itoa(i)}) } + _, err := s.ds.UpsertSoftwareCPEs(context.Background(), cpes) + require.NoError(t, err) + // Reload software to load GeneratedCPEID require.NoError(t, s.ds.LoadHostSoftware(context.Background(), hosts[0], false)) - var vulns []fleet.SoftwareVulnerability - for i, sw := range hosts[0].Software[:10] { - vulns = append(vulns, fleet.SoftwareVulnerability{ - SoftwareID: sw.ID, - CVE: fmt.Sprintf("cve-123-123-%03d", i), - }) - } // add CVEs for the first 10 software, which are the least used (lower hosts_count) - n, err := s.ds.InsertSoftwareVulnerabilities(context.Background(), vulns, fleet.NVDSource) - require.NoError(t, err) - require.Equal(t, 10, int(n)) + for i, sw := range hosts[0].Software[:10] { + inserted, err := s.ds.InsertSoftwareVulnerability(context.Background(), fleet.SoftwareVulnerability{ + SoftwareID: sw.ID, + CVE: fmt.Sprintf("cve-123-123-%03d", i), + }, fleet.NVDSource) + require.NoError(t, err) + require.True(t, inserted) + } // create a team and make the last 3 hosts part of it (meaning 3 that use // sws[19], 2 for sws[18], and 1 for sws[17]) diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 44f346eacf..bba1677fed 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -2465,16 +2465,14 @@ func (s *integrationEnterpriseTestSuite) TestListSoftware() { bar = host.Software[1] } - n, err := s.ds.InsertSoftwareVulnerabilities( - ctx, []fleet.SoftwareVulnerability{ - { - SoftwareID: bar.ID, - CVE: "cve-123", - }, + inserted, err := s.ds.InsertSoftwareVulnerability( + ctx, fleet.SoftwareVulnerability{ + SoftwareID: bar.ID, + CVE: "cve-123", }, fleet.NVDSource, ) require.NoError(t, err) - require.Equal(t, 1, int(n)) + require.True(t, inserted) require.NoError(t, s.ds.InsertCVEMeta(ctx, []fleet.CVEMeta{{ CVE: "cve-123", diff --git a/server/vulnerabilities/macoffice/analyzer.go b/server/vulnerabilities/macoffice/analyzer.go index 0c9ad102fd..985fab8440 100644 --- a/server/vulnerabilities/macoffice/analyzer.go +++ b/server/vulnerabilities/macoffice/analyzer.go @@ -119,14 +119,16 @@ func updateVulnsInDB( } inserted := make([]fleet.SoftwareVulnerability, 0, len(toInsertSet)) - err = utils.BatchProcess(toInsertSet, func(v []fleet.SoftwareVulnerability) error { - n, err := ds.InsertSoftwareVulnerabilities(ctx, v, fleet.MacOfficeReleaseNotesSource) - if err != nil { - return err - } + err = utils.BatchProcess(toInsertSet, func(vulns []fleet.SoftwareVulnerability) error { + for _, v := range vulns { + ok, err := ds.InsertSoftwareVulnerability(ctx, v, fleet.MacOfficeReleaseNotesSource) + if err != nil { + return err + } - if n > 0 { - inserted = append(inserted, v...) + if ok { + inserted = append(inserted, v) + } } return nil @@ -157,7 +159,8 @@ func Analyze( return nil, nil } - iter, err := ds.ListSoftwareBySourceIter(ctx, []string{"apps"}) + queryParams := fleet.SoftwareIterQueryOptions{IncludedSources: []string{"apps"}} + iter, err := ds.AllSoftwareIterator(ctx, queryParams) if err != nil { return nil, err } diff --git a/server/vulnerabilities/macoffice/analyzer_test.go b/server/vulnerabilities/macoffice/analyzer_test.go index a09b6df528..b4c7831043 100644 --- a/server/vulnerabilities/macoffice/analyzer_test.go +++ b/server/vulnerabilities/macoffice/analyzer_test.go @@ -38,8 +38,8 @@ func TestAnalyzer(t *testing.T) { ds.DeleteSoftwareVulnerabilitiesFunc = func(ctx context.Context, vulnerabilities []fleet.SoftwareVulnerability) error { return errors.New("some error") } - ds.InsertSoftwareVulnerabilitiesFunc = func(ctx context.Context, vulns []fleet.SoftwareVulnerability, source fleet.VulnerabilitySource) (int64, error) { - return 0, nil + ds.InsertSoftwareVulnerabilityFunc = func(ctx context.Context, vuln fleet.SoftwareVulnerability, source fleet.VulnerabilitySource) (bool, error) { + return false, nil } vulns, err := updateVulnsInDB(ctx, ds, nil, nil) @@ -56,8 +56,8 @@ func TestAnalyzer(t *testing.T) { ds.DeleteSoftwareVulnerabilitiesFunc = func(ctx context.Context, vulnerabilities []fleet.SoftwareVulnerability) error { return nil } - ds.InsertSoftwareVulnerabilitiesFunc = func(ctx context.Context, vulns []fleet.SoftwareVulnerability, source fleet.VulnerabilitySource) (int64, error) { - return 0, errors.New("some error") + ds.InsertSoftwareVulnerabilityFunc = func(ctx context.Context, vuln fleet.SoftwareVulnerability, source fleet.VulnerabilitySource) (bool, error) { + return false, errors.New("some error") } vulns, err := updateVulnsInDB(ctx, ds, detected, nil) diff --git a/server/vulnerabilities/macoffice/integration_analyzer_test.go b/server/vulnerabilities/macoffice/integration_analyzer_test.go index 3d14dcec00..bd347c5195 100644 --- a/server/vulnerabilities/macoffice/integration_analyzer_test.go +++ b/server/vulnerabilities/macoffice/integration_analyzer_test.go @@ -186,10 +186,16 @@ func TestIntegrationAnalyzer(t *testing.T) { } // These 'old' vulnerabilities should be cleared out... - _, err := ds.InsertSoftwareVulnerabilities(ctx, []fleet.SoftwareVulnerability{ - {SoftwareID: word.ID, CVE: "3000-3000"}, - {SoftwareID: powerpoint.ID, CVE: "4000-3000"}, + ok, err := ds.InsertSoftwareVulnerability(ctx, fleet.SoftwareVulnerability{ + SoftwareID: word.ID, CVE: "3000-3000", }, fleet.MacOfficeReleaseNotesSource) + require.True(t, ok) + require.NoError(t, err) + + ok, err = ds.InsertSoftwareVulnerability(ctx, fleet.SoftwareVulnerability{ + SoftwareID: powerpoint.ID, CVE: "4000-3000", + }, fleet.MacOfficeReleaseNotesSource) + require.True(t, ok) require.NoError(t, err) vulns, err := macoffice.Analyze(ctx, ds, vulnPath, true) diff --git a/server/vulnerabilities/nvd/cpe.go b/server/vulnerabilities/nvd/cpe.go index 2a1e856602..28ae9da80d 100644 --- a/server/vulnerabilities/nvd/cpe.go +++ b/server/vulnerabilities/nvd/cpe.go @@ -297,6 +297,48 @@ func CPEFromSoftware(db *sqlx.DB, software *fleet.Software, translations CPETran return "", nil } +func consumeCPEBuffer( + ctx context.Context, + ds fleet.Datastore, + logger kitlog.Logger, + batch []fleet.SoftwareCPE, +) error { + var toDelete []fleet.SoftwareCPE + var toUpsert []fleet.SoftwareCPE + + for i := range batch { + // This could be because of a new translation rule or because we fixed a bug with the CPE + // detection process + if batch[i].CPE == "" { + toDelete = append(toDelete, batch[i]) + continue + } + toUpsert = append(toUpsert, batch[i]) + } + + if len(toUpsert) != 0 { + upserted, err := ds.UpsertSoftwareCPEs(ctx, toUpsert) + if err != nil { + return err + } + if int(upserted) != len(toUpsert) { + level.Debug(logger).Log("toUpsert", len(toUpsert), "upserted", upserted) + } + } + + if len(toDelete) != 0 { + deleted, err := ds.DeleteSoftwareCPEs(ctx, toDelete) + if err != nil { + return err + } + if int(deleted) != len(toDelete) { + level.Debug(logger).Log("toDelete", len(toDelete), "deleted", deleted) + } + } + + return nil +} + func TranslateSoftwareToCPE( ctx context.Context, ds fleet.Datastore, @@ -306,9 +348,14 @@ func TranslateSoftwareToCPE( dbPath := filepath.Join(vulnPath, cpeDBFilename) // Skip software from sources for which we will be using OVAL for vulnerability detection. - iterator, err := ds.AllSoftwareWithoutCPEIterator(ctx, oval.SupportedSoftwareSources) + iterator, err := ds.AllSoftwareIterator( + ctx, + fleet.SoftwareIterQueryOptions{ + ExcludedSources: oval.SupportedSoftwareSources, + }, + ) if err != nil { - return ctxerr.Wrap(ctx, err, "all software iterator") + return ctxerr.Wrap(ctx, err, "software iterator") } defer iterator.Close() @@ -326,6 +373,9 @@ func TranslateSoftwareToCPE( reCache := newRegexpCache() + var buffer []fleet.SoftwareCPE + bufferMaxSize := 500 + for iterator.Next() { software, err := iterator.Value() if err != nil { @@ -336,14 +386,26 @@ func TranslateSoftwareToCPE( level.Error(logger).Log("software->cpe", "error translating to CPE, skipping...", "err", err) continue } - if cpe == "" { + if cpe == software.GenerateCPE { continue } - err = ds.AddCPEForSoftware(ctx, *software, cpe) - if err != nil { - return ctxerr.Wrap(ctx, err, "inserting cpe") + + buffer = append(buffer, fleet.SoftwareCPE{SoftwareID: software.ID, CPE: cpe}) + if len(buffer) == bufferMaxSize { + if err = consumeCPEBuffer(ctx, ds, logger, buffer); err != nil { + return ctxerr.Wrap(ctx, err, "inserting cpe") + } + buffer = nil } } + if err = consumeCPEBuffer(ctx, ds, logger, buffer); err != nil { + return ctxerr.Wrap(ctx, err, "inserting cpe") + } + + if err := iterator.Err(); err != nil { + return ctxerr.Wrap(ctx, err, "iterator contains error at the end of iteration") + } + return nil } diff --git a/server/vulnerabilities/nvd/cpe_test.go b/server/vulnerabilities/nvd/cpe_test.go index 4f7dc1ef1d..1bba7244b8 100644 --- a/server/vulnerabilities/nvd/cpe_test.go +++ b/server/vulnerabilities/nvd/cpe_test.go @@ -237,6 +237,65 @@ func (f *fakeSoftwareIterator) Value() (*fleet.Software, error) { func (f *fakeSoftwareIterator) Err() error { return nil } func (f *fakeSoftwareIterator) Close() error { f.closed = true; return nil } +func TestConsumeCPEBuffer(t *testing.T) { + ctx := context.Background() + + t.Run("empty buffer", func(t *testing.T) { + var upserted []fleet.SoftwareCPE + var deleted []fleet.SoftwareCPE + + ds := new(mock.Store) + ds.UpsertSoftwareCPEsFunc = func(ctx context.Context, cpes []fleet.SoftwareCPE) (int64, error) { + upserted = append(upserted, cpes...) + return int64(len(upserted)), nil + } + + ds.DeleteSoftwareCPEsFunc = func(ctx context.Context, cpes []fleet.SoftwareCPE) (int64, error) { + deleted = append(deleted, cpes...) + return int64(len(deleted)), nil + } + err := consumeCPEBuffer(ctx, ds, nil, nil) + require.NoError(t, err) + require.Empty(t, upserted) + require.Empty(t, deleted) + }) + + t.Run("inserts and deletes accordantly", func(t *testing.T) { + var upserted []fleet.SoftwareCPE + var deleted []fleet.SoftwareCPE + + ds := new(mock.Store) + ds.UpsertSoftwareCPEsFunc = func(ctx context.Context, cpes []fleet.SoftwareCPE) (int64, error) { + upserted = append(upserted, cpes...) + return int64(len(upserted)), nil + } + + ds.DeleteSoftwareCPEsFunc = func(ctx context.Context, cpes []fleet.SoftwareCPE) (int64, error) { + deleted = append(deleted, cpes...) + return int64(len(deleted)), nil + } + + cpes := []fleet.SoftwareCPE{ + { + SoftwareID: 1, + CPE: "", + }, + { + SoftwareID: 2, + CPE: "cpe-1", + }, + } + + err := consumeCPEBuffer(ctx, ds, nil, cpes) + require.NoError(t, err) + require.Equal(t, len(upserted), 1) + require.Equal(t, upserted[0].CPE, cpes[1].CPE) + + require.Equal(t, len(deleted), 1) + require.Equal(t, deleted[0].CPE, cpes[0].CPE) + }) +} + func TestTranslateSoftwareToCPE(t *testing.T) { nettest.Run(t) @@ -246,9 +305,11 @@ func TestTranslateSoftwareToCPE(t *testing.T) { var cpes []string - ds.AddCPEForSoftwareFunc = func(ctx context.Context, software fleet.Software, cpe string) error { - cpes = append(cpes, cpe) - return nil + ds.UpsertSoftwareCPEsFunc = func(ctx context.Context, vals []fleet.SoftwareCPE) (int64, error) { + for _, v := range vals { + cpes = append(cpes, v.CPE) + } + return int64(len(vals)), nil } iterator := &fakeSoftwareIterator{ @@ -266,11 +327,22 @@ func TestTranslateSoftwareToCPE(t *testing.T) { Version: "0.3", BundleIdentifier: "vendor2", Source: "apps", + GenerateCPE: "something_wrong", + }, + // For the following software entry, the matched cpe will match 'GenerateCPE', so we are + // adding it to test that that 'UpsertSoftwareCPEs' will only be called iff software.GenerateCPE != detected CPE. + { + ID: 3, + Name: "Product2", + Version: "0.3", + BundleIdentifier: "vendor2", + Source: "apps", + GenerateCPE: "cpe:2.3:a:vendor2:product4:0.3:*:*:*:*:macos:*:*", }, }, } - ds.AllSoftwareWithoutCPEIteratorFunc = func(ctx context.Context, excludedPlatforms []string) (fleet.SoftwareIterator, error) { + ds.AllSoftwareIteratorFunc = func(ctx context.Context, q fleet.SoftwareIterQueryOptions) (fleet.SoftwareIterator, error) { return iterator, nil } @@ -359,7 +431,8 @@ func TestCPEFromSoftwareIntegration(t *testing.T) { Version: "22.002.20191", Vendor: "", BundleIdentifier: "com.adobe.Reader", - }, cpe: "cpe:2.3:a:adobe:acrobat_reader_dc:22.002.20191:*:*:*:*:macos:*:*", + }, + cpe: "cpe:2.3:a:adobe:acrobat_reader_dc:22.002.20191:*:*:*:*:macos:*:*", }, { software: fleet.Software{ diff --git a/server/vulnerabilities/nvd/cve.go b/server/vulnerabilities/nvd/cve.go index 0601839890..92c01d4817 100644 --- a/server/vulnerabilities/nvd/cve.go +++ b/server/vulnerabilities/nvd/cve.go @@ -100,6 +100,7 @@ func TranslateCPEToCVE( vulnPath string, logger kitlog.Logger, collectVulns bool, + periodicity time.Duration, ) ([]fleet.SoftwareVulnerability, error) { files, err := getNVDCVEFeedFiles(vulnPath) if err != nil { @@ -147,20 +148,27 @@ func TranslateCPEToCVE( var newVulns []fleet.SoftwareVulnerability for _, vuln := range vulns { - newCount, err := ds.InsertSoftwareVulnerabilities(ctx, []fleet.SoftwareVulnerability{vuln}, fleet.NVDSource) + ok, err := ds.InsertSoftwareVulnerability(ctx, vuln, fleet.NVDSource) if err != nil { level.Error(logger).Log("cpe processing", "error", "err", err) continue } - // collect vuln only if newCount > 0, otherwise we would send + // collect vuln only if inserted, otherwise we would send // webhook requests for the same vulnerability over and over again until // it is older than 2 days. - if collectVulns && newCount > 0 { + if collectVulns && ok { newVulns = append(newVulns, vuln) } } + // Delete any stale vulnerabilities. A vulnerability is stale iff the last time it was + // updated was more than `2 * periodicity` ago. This assumes that the whole vulnerability + // process completes in less than `periodicity` units of time. + if err = ds.DeleteOutOfDateVulnerabilities(ctx, fleet.NVDSource, 2*periodicity); err != nil { + level.Error(logger).Log("msg", "error deleting out of date vulnerabilities", "err", err) + } + return newVulns, nil } diff --git a/server/vulnerabilities/nvd/cve_test.go b/server/vulnerabilities/nvd/cve_test.go index 00b27d343a..8069f9cac0 100644 --- a/server/vulnerabilities/nvd/cve_test.go +++ b/server/vulnerabilities/nvd/cve_test.go @@ -10,6 +10,7 @@ import ( "strings" "sync" "testing" + "time" "github.com/fleetdm/fleet/v4/pkg/nettest" "github.com/fleetdm/fleet/v4/server/fleet" @@ -169,10 +170,10 @@ func (d *threadSafeDSMock) ListSoftwareCPEs(ctx context.Context) ([]fleet.Softwa return d.Store.ListSoftwareCPEs(ctx) } -func (d *threadSafeDSMock) InsertSoftwareVulnerabilities(ctx context.Context, vulns []fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (int64, error) { +func (d *threadSafeDSMock) InsertSoftwareVulnerability(ctx context.Context, vuln fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (bool, error) { d.mu.Lock() defer d.mu.Unlock() - return d.Store.InsertSoftwareVulnerabilities(ctx, vulns, src) + return d.Store.InsertSoftwareVulnerability(ctx, vuln, src) } func TestTranslateCPEToCVE(t *testing.T) { @@ -199,18 +200,17 @@ func TestTranslateCPEToCVE(t *testing.T) { cveLock := &sync.Mutex{} var cvesFound []string - ds.InsertSoftwareVulnerabilitiesFunc = func(ctx context.Context, vulns []fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (int64, error) { + ds.InsertSoftwareVulnerabilityFunc = func(ctx context.Context, vuln fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (bool, error) { cveLock.Lock() defer cveLock.Unlock() - - for _, v := range vulns { - cvesFound = append(cvesFound, v.CVE) - } - - return 0, nil + cvesFound = append(cvesFound, vuln.CVE) + return false, nil + } + ds.DeleteOutOfDateVulnerabilitiesFunc = func(ctx context.Context, source fleet.VulnerabilitySource, duration time.Duration) error { + return nil } - _, err := TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewLogfmtLogger(os.Stdout), false) + _, err := TranslateCPEToCVE(ctx, ds, tempDir, kitlog.NewLogfmtLogger(os.Stdout), false, 1*time.Hour) require.NoError(t, err) printMemUsage() @@ -226,6 +226,8 @@ func TestTranslateCPEToCVE(t *testing.T) { // Check for exact match of CVEs found. require.ElementsMatch(t, cvesFound, tt.cves, tt.cpe) } + + require.True(t, ds.DeleteOutOfDateVulnerabilitiesFuncInvoked) }) } @@ -241,10 +243,10 @@ func TestTranslateCPEToCVE(t *testing.T) { return softwareCPEs, nil } - ds.InsertSoftwareVulnerabilitiesFunc = func(ctx context.Context, vulns []fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (int64, error) { - return 1, nil + ds.InsertSoftwareVulnerabilityFunc = func(ctx context.Context, vuln fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (bool, error) { + return true, nil } - recent, err := TranslateCPEToCVE(ctx, safeDS, tempDir, kitlog.NewNopLogger(), true) + recent, err := TranslateCPEToCVE(ctx, safeDS, tempDir, kitlog.NewNopLogger(), true, 1*time.Hour) require.NoError(t, err) byCPE := make(map[uint]int) @@ -259,12 +261,12 @@ func TestTranslateCPEToCVE(t *testing.T) { assert.Greater(t, byCPE[softwareCPEs[1].SoftwareID], 280, "mozilla firefox CVEs") assert.Greater(t, byCPE[softwareCPEs[2].SoftwareID], 10, "curl CVEs") - // call it again but now return 0 from this call, simulating CVE-CPE pairs + // call it again but now return false from this call, simulating CVE-CPE pairs // that already existed in the DB. - ds.InsertSoftwareVulnerabilitiesFunc = func(ctx context.Context, vulns []fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (int64, error) { - return 0, nil + ds.InsertSoftwareVulnerabilityFunc = func(ctx context.Context, vuln fleet.SoftwareVulnerability, src fleet.VulnerabilitySource) (bool, error) { + return false, nil } - recent, err = TranslateCPEToCVE(ctx, safeDS, tempDir, kitlog.NewNopLogger(), true) + recent, err = TranslateCPEToCVE(ctx, safeDS, tempDir, kitlog.NewNopLogger(), true, 1*time.Hour) require.NoError(t, err) // no recent vulnerability should be reported diff --git a/server/vulnerabilities/oval/analyzer.go b/server/vulnerabilities/oval/analyzer.go index db04cc9e8e..2c56c2b03f 100644 --- a/server/vulnerabilities/oval/analyzer.go +++ b/server/vulnerabilities/oval/analyzer.go @@ -102,16 +102,17 @@ func Analyze( inserted = make([]fleet.SoftwareVulnerability, 0, len(toInsertSet)) } - err = utils.BatchProcess(toInsertSet, func(v []fleet.SoftwareVulnerability) error { - n, err := ds.InsertSoftwareVulnerabilities(ctx, v, source) - if err != nil { - return err - } + err = utils.BatchProcess(toInsertSet, func(vulns []fleet.SoftwareVulnerability) error { + for _, v := range vulns { + ok, err := ds.InsertSoftwareVulnerability(ctx, v, source) + if err != nil { + return err + } - if collectVulns && n > 0 { - inserted = append(inserted, v...) + if collectVulns && ok { + inserted = append(inserted, v) + } } - return nil }, vulnBatchSize) if err != nil { diff --git a/server/vulnerabilities/oval/analyzer_test.go b/server/vulnerabilities/oval/analyzer_test.go index 6b4f5369e3..445c24d2e5 100644 --- a/server/vulnerabilities/oval/analyzer_test.go +++ b/server/vulnerabilities/oval/analyzer_test.go @@ -90,10 +90,12 @@ func loadSoftware( err = ds.LoadHostSoftware(ctx, h, false) require.NoError(t, err) + var cpes []fleet.SoftwareCPE for _, s := range h.Software { - err = ds.AddCPEForSoftware(ctx, s, fmt.Sprintf("%s-%s", s.Name, s.Version)) - require.NoError(t, err) + cpes = append(cpes, fleet.SoftwareCPE{SoftwareID: s.ID, CPE: fmt.Sprintf("%s-%s", s.Name, s.Version)}) } + _, err = ds.UpsertSoftwareCPEs(ctx, cpes) + require.NoError(t, err) return h }