From eada583ff10eee983426cce3edcb11e0ced66ca6 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Mon, 20 Nov 2023 16:10:00 -0600 Subject: [PATCH] Updating CPE generator to use new NVD API. (#15018) Loom explaining changes (hit 5 min limit): https://www.loom.com/share/e59b63bf638e4d9cad7984ef589b878d?sid=111fff75-115a-4a44-ae4f-6f25fede0d51 #14887 - [x] Need to merge fleetdm/nvd PR https://github.com/fleetdm/nvd/pull/25 before this one. # Checklist for submitter - [x] Added/updated tests - [x] Manual QA for all new/changed functionality - Manually tested (with corresponding fleetdm/fleet changes) in my personal fork: https://github.com/getvictor/nvd/releases # QA Plan (must be done before merging this PR, and after merging the nvd PR) - [ ] Fork https://github.com/fleetdm/nvd and point `generate.yml` to this branch. [example](https://github.com/getvictor/nvd/blob/9d8e54930bc174b00cc2daa70f55cabf0f9dba6e/.github/workflows/generate.yml#L26) - [ ] Add NVD_API_KEY to nvd secrets, and run the the nvd generate GitHub action. Get key: https://nvd.nist.gov/developers/request-an-api-key - [ ] Compare the generated `cpe-###.sqlite.gz` to the previous one. One way is to open it up with sqlite3 and `select * from cpe_2 order by cpe23;` and dump results to a CSV file. Known differences are: - New file has ~2,500 more records - Backslashes are handled differently for `Backpack\CRUD` and `Philips In.Sight B120\37` products -- not a new issue since we do not support those products right now - `cpe:2.3:a:moodle:moodle:4.2.0:*:*:*:*:*:*:*` -- this appears OK. Also, it is a PHP plugin, and we don't support these currently. - [ ] Record the existing vulnerabilities of current hosts. - [ ] Stop any running fleet server. Delete `/tmp/vulndbs/cpe.sqlite`. Can also delete other files there, or not delete this file -- it should be overwritten by the new file. Also delete all rows in software_cpe and software_cve DB tables. (Or can just spin up a fresh fleet server with fresh DB, and re-enroll hosts (after setting the new env variable below)) - [ ] Find the path to the generated `cpe-###.sqlite.gz` file - [ ] Set `FLEET_VULNERABILITIES_CPE_DATABASE_URL` environment variable to the above path, and start fleet server. - [ ] After server's vulnerabilities cron job runs, the new vulnerabilities should match the previous vulnerabilities --- cmd/cpe/generate.go | 218 +++++++++++++++++++------ cmd/cpe/generate_test.go | 89 ++++++++++ cmd/cpe/testdata/test1.golden | 1 + cmd/cpe/testdata/test1.json | 118 +++++++++++++ cmd/cpe/testdata/test2.golden | 1 + cmd/cpe/testdata/test2.json | 60 +++++++ go.mod | 2 + go.sum | 4 + server/vulnerabilities/nvd/cpe_test.go | 10 +- server/vulnerabilities/nvd/db.go | 4 +- 10 files changed, 451 insertions(+), 56 deletions(-) create mode 100644 cmd/cpe/generate_test.go create mode 100644 cmd/cpe/testdata/test1.golden create mode 100644 cmd/cpe/testdata/test1.json create mode 100644 cmd/cpe/testdata/test2.golden create mode 100644 cmd/cpe/testdata/test2.json diff --git a/cmd/cpe/generate.go b/cmd/cpe/generate.go index 815bfd7ade..1f4dcfdf55 100644 --- a/cmd/cpe/generate.go +++ b/cmd/cpe/generate.go @@ -2,98 +2,187 @@ package main import ( "compress/gzip" - "flag" + "crypto/sha256" "fmt" + "github.com/facebookincubator/nvdtools/cpedict" + "github.com/facebookincubator/nvdtools/wfn" + "github.com/fleetdm/fleet/v4/pkg/fleethttp" + "github.com/fleetdm/fleet/v4/server/ptr" + "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd" + "github.com/pandatix/nvdapi/common" + "github.com/pandatix/nvdapi/v2" "io" - "net/http" + "log" + "log/slog" "os" "path/filepath" "strings" - - "github.com/facebookincubator/nvdtools/cpedict" - "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd" + "time" ) -func panicif(err error) { +const ( + httpClientTimeout = 2 * time.Minute + waitTimeBetweenRequests = 6 * time.Second + waitTimeForRetry = 30 * time.Second + maxRetryAttempts = 10 + apiKeyEnvVar = "NVD_API_KEY" //nolint:gosec +) + +func panicIf(err error) { if err != nil { panic(err) } } func main() { - var verbose bool - flag.BoolVar(&verbose, "verbose", false, "Sets verbose mode") - flag.Parse() + apiKey := os.Getenv(apiKeyEnvVar) - dbPath := cpe() + logHandler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}) + slog.SetDefault(slog.New(logHandler)) - fmt.Printf("Sqlite file %s size: %.2f MB\n", dbPath, getSizeMB(dbPath)) + if apiKey == "" { + log.Fatal(fmt.Sprintf("Must set %v environment variable", apiKeyEnvVar)) + } - fmt.Println("Compressing DB...") + cwd, err := os.Getwd() + panicIf(err) + slog.Info(fmt.Sprintf("CWD: %v", cwd)) + + client := fleethttp.NewClient(fleethttp.WithTimeout(httpClientTimeout)) + dbPath := getCPEs(client, apiKey, cwd) + + slog.Info(fmt.Sprintf("Sqlite file %s size: %.2f MB\n", dbPath, getSizeMB(dbPath))) + + slog.Info("Compressing DB...") compressedPath, err := compress(dbPath) - panicif(err) + panicIf(err) - fmt.Printf("Final compressed file %s size: %.2f MB\n", compressedPath, getSizeMB(compressedPath)) - fmt.Println("Done.") + slog.Info("Calculating SHA256...") + compressedPath, err = addSHA256(compressedPath) + panicIf(err) + + slog.Info(fmt.Sprintf("Final compressed file %s size: %.2f MB\n", compressedPath, getSizeMB(compressedPath))) + slog.Info("Done.") } func getSizeMB(path string) float64 { info, err := os.Stat(path) - panicif(err) + panicIf(err) return float64(info.Size()) / 1024.0 / 1024.0 } -func cpe() string { - fmt.Println("Starting CPE sqlite generation...") +func getCPEs(client common.HTTPClient, apiKey string, resultPath string) string { + slog.Info("Fetching CPEs from NVD...") - cwd, err := os.Getwd() - panicif(err) - fmt.Println("CWD:", cwd) + nvdClient, err := nvdapi.NewNVDClient(client, apiKey) + panicIf(err) - resp, err := http.Get("https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz") - panicif(err) - defer resp.Body.Close() + var cpes []cpedict.CPEItem + retryAttempts := 0 - remoteEtag := getSanitizedEtag(resp) - fmt.Println("Got ETag:", remoteEtag) + totalResults := 1 + for startIndex := 0; startIndex < totalResults; { + cpeResponse, err := nvdapi.GetCPEs(nvdClient, nvdapi.GetCPEsParams{StartIndex: ptr.Int(startIndex)}) + if err != nil { + if retryAttempts > maxRetryAttempts { + panicIf(err) + } + slog.Warn(fmt.Sprintf("NVD request returned error:'%v' Retrying in %v", err.Error(), waitTimeForRetry.String())) + retryAttempts++ + time.Sleep(waitTimeForRetry) + continue + } + retryAttempts = 0 + totalResults = cpeResponse.TotalResults + slog.Info(fmt.Sprintf("Got %v results", cpeResponse.ResultsPerPage)) + startIndex += cpeResponse.ResultsPerPage + for _, product := range cpeResponse.Products { + cpes = append(cpes, convertToCPEItem(product.CPE)) + } + if startIndex < totalResults { + // NVD API recommendation to sleep between requests: https://nvd.nist.gov/developers/api-workflows + time.Sleep(waitTimeBetweenRequests) + slog.Info(fmt.Sprintf("Fetching index %v out of %v", startIndex, totalResults)) + } + } - gr, err := gzip.NewReader(resp.Body) - panicif(err) - defer gr.Close() + // Sanity check + if totalResults <= 1 || len(cpes) != totalResults { + log.Fatal(fmt.Sprintf("Invalid number of expected results:%v or actual results:%v", totalResults, len(cpes))) + } - cpeDict, err := cpedict.Decode(gr) - panicif(err) + slog.Info("Generating CPE sqlite DB...") - fmt.Println("Generating DB...") - dbPath := filepath.Join(cwd, fmt.Sprintf("cpe-%s.sqlite", remoteEtag)) - err = nvd.GenerateCPEDB(dbPath, cpeDict) - panicif(err) - - file, err := os.Create(filepath.Join(cwd, "etagenv")) - panicif(err) - _, err = file.WriteString(fmt.Sprintf(`ETAG=%s`, remoteEtag)) - panicif(err) - file.Close() + dbPath := filepath.Join(resultPath, fmt.Sprint("cpe.sqlite")) + err = nvd.GenerateCPEDB(dbPath, cpes) + panicIf(err) return dbPath } +func convertToCPEItem(in nvdapi.CPE) (out cpedict.CPEItem) { + out = cpedict.CPEItem{} + + // CPE name + wfName, err := wfn.Parse(in.CPEName) + panicIf(err) + out.CPE23 = cpedict.CPE23Item{ + Name: cpedict.NamePattern(*wfName), + } + + // Deprecations + out.Deprecated = in.Deprecated + if in.Deprecated { + out.CPE23.Deprecation = &cpedict.Deprecation{} + for _, item := range in.DeprecatedBy { + deprecatorName, err := wfn.Parse(*item.CPEName) + panicIf(err) + deprecatorInfo := cpedict.DeprecatedInfo{ + Name: cpedict.NamePattern(*deprecatorName), + } + out.CPE23.Deprecation.DeprecatedBy = append(out.CPE23.Deprecation.DeprecatedBy, deprecatorInfo) + } + } + + // Title + out.Title = cpedict.TextType{} + for _, title := range in.Titles { + // only using English language + if title.Lang == "en" { + out.Title["en-US"] = title.Title + break + } + } + + // The following fields are not needed by subsequent code: + // out.DeprecatedBy + // out.DeprecationDate + // out.Notes + // out.References + return out +} + func compress(path string) (string, error) { compressedPath := fmt.Sprintf("%s.gz", path) compressedDB, err := os.Create(compressedPath) if err != nil { return "", err } - defer compressedDB.Close() + defer closeFile(compressedDB) db, err := os.Open(path) if err != nil { return "", err } - defer db.Close() + defer closeFile(db) w := gzip.NewWriter(compressedDB) - defer w.Close() + defer func(w *gzip.Writer) { + err := w.Close() + if err != nil { + slog.Error(fmt.Sprintf("Could not close gzip.Writer: %v", err.Error())) + } + }(w) _, err = io.Copy(w, db) if err != nil { @@ -102,9 +191,40 @@ func compress(path string) (string, error) { return compressedPath, nil } -func getSanitizedEtag(resp *http.Response) string { - etag := resp.Header.Get("Etag") - etag = strings.TrimPrefix(strings.TrimSuffix(etag, `"`), `"`) - etag = strings.Replace(etag, ":", "", -1) - return etag +// addSHA256 adds the file's SHA256 checksum to its name +func addSHA256(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer closeFile(file) + + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + + newPath, err := replaceLast(path, "cpe.sqlite.gz", fmt.Sprintf("cpe-%x.sqlite.gz", hash.Sum(nil))) + if err != nil { + return "", err + } + + err = os.Rename(path, newPath) + return newPath, err +} + +// replaceLast replaces the last occurrence of string +func replaceLast(s, old, new string) (string, error) { + i := strings.LastIndex(s, old) + if i == -1 { + return "", fmt.Errorf("substring:%v not found in string:%v", old, s) + } + return s[:i] + new + s[i+len(old):], nil +} + +func closeFile(file *os.File) { + err := file.Close() + if err != nil { + slog.Error(fmt.Sprintf("Could not close file %v: %v", file.Name(), err.Error())) + } } diff --git a/cmd/cpe/generate_test.go b/cmd/cpe/generate_test.go new file mode 100644 index 0000000000..83a783da39 --- /dev/null +++ b/cmd/cpe/generate_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "fmt" + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/require" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +type mockClient struct { + response *http.Response +} + +func (m *mockClient) Do(*http.Request) (*http.Response, error) { + return m.response, nil +} + +func TestCPEDB(t *testing.T) { + + // Find the paths of all input files in the testdata directory. + paths, err := filepath.Glob(filepath.Join("testdata", "*.json")) + if err != nil { + t.Fatal(err) + } + + for _, p := range paths { + path := p + _, filename := filepath.Split(path) + testName := filename[:len(filename)-len(filepath.Ext(path))] + + // Each path turns into a test: the test name is the filename without the extension. + t.Run( + testName, func(t *testing.T) { + t.Parallel() + json, err := os.ReadFile(path) + require.NoError(t, err) + + // Set up HTTP response + recorder := httptest.NewRecorder() + recorder.Header().Add("Content-Type", "application/json") + _, _ = recorder.WriteString(string(json)) + expectedResponse := recorder.Result() + + // Create an HTTP client + client := mockClient{response: expectedResponse} + + // Temporary directory, which will be automatically cleaned up + dir := t.TempDir() + + // Call the function under test + dbPath := getCPEs(&client, "API_KEY", dir) + + // Open up the created DB and get the rows + db, err := sqlx.Open("sqlite3", dbPath) + require.NoError(t, err) + defer db.Close() + rows, err := db.Query("SELECT * FROM cpe_2") + require.NoError(t, err) + require.NoError(t, rows.Err()) + defer rows.Close() + + // Convert rows to string for comparison + var result [][]string + cols, _ := rows.Columns() + for rows.Next() { + // Setting up for converting row to a string + pointers := make([]interface{}, len(cols)) + container := make([]string, len(cols)) + for i := range pointers { + pointers[i] = &container[i] + } + + require.NoError(t, rows.Scan(pointers...)) + result = append(result, container) + } + + // Compare result to the .golden file + goldenFile := filepath.Join("testdata", testName+".golden") + golden, err := os.ReadFile(goldenFile) + require.NoError(t, err) + require.Equal(t, string(golden), fmt.Sprintf("%s", result)) + }, + ) + } +} diff --git a/cmd/cpe/testdata/test1.golden b/cmd/cpe/testdata/test1.golden new file mode 100644 index 0000000000..7661509716 --- /dev/null +++ b/cmd/cpe/testdata/test1.golden @@ -0,0 +1 @@ +[[cpe:2.3:a:hp:radia_notify_daemon:-:*:*:*:*:*:*:* HP Radia Notify Daemon hp radia_notify_daemon - false] [cpe:2.3:a:hp:sanworks:-:*:*:*:*:*:*:* HP SANworks hp sanworks - false] [cpe:2.3:a:hp:scanjet_utilities:-:*:*:*:*:*:*:* HP Scanjet Utilities hp scanjet_utilities - false] [cpe:2.3:a:hp:secure_web_console:-:*:*:*:*:*:*:* HP Secure Web Console hp secure_web_console - false] [cpe:2.3:a:hp:sendmail:-:*:*:*:*:*:*:* HP sendmail hp sendmail - false] [cpe:2.3:o:linux:linux_kernel:2.6.2:*:*:*:*:*:*:* Linux Kernel 2.6.2 linux linux_kernel 2.6.2 true]] \ No newline at end of file diff --git a/cmd/cpe/testdata/test1.json b/cmd/cpe/testdata/test1.json new file mode 100644 index 0000000000..98d5d049c1 --- /dev/null +++ b/cmd/cpe/testdata/test1.json @@ -0,0 +1,118 @@ +{ + "resultsPerPage": 6, + "startIndex": 0, + "totalResults": 6, + "format": "NVD_CPE", + "version": "2.0", + "timestamp": "2023-11-07T20:02:39.860", + "products": [ + { + "cpe": { + "deprecated": false, + "cpeName": "cpe:2.3:a:hp:radia_notify_daemon:-:*:*:*:*:*:*:*", + "cpeNameId": "9E1C1A60-AFDF-4F21-94D9-078EDA0DECEC", + "lastModified": "2007-09-14T17:36:49.090", + "created": "2007-08-23T21:05:57.937", + "titles": [ + { + "title": "HP Radia Notify Daemon", + "lang": "en" + } + ] + } + }, + { + "cpe": { + "deprecated": false, + "cpeName": "cpe:2.3:a:hp:sanworks:-:*:*:*:*:*:*:*", + "cpeNameId": "6C7DD0D6-4DB8-4AE7-BF44-11A40253543E", + "lastModified": "2008-04-15T22:37:41.817", + "created": "2007-08-23T21:05:57.937", + "titles": [ + { + "title": "HP SANworks", + "lang": "en" + } + ] + } + }, + { + "cpe": { + "deprecated": false, + "cpeName": "cpe:2.3:a:hp:scanjet_utilities:-:*:*:*:*:*:*:*", + "cpeNameId": "87E045F2-D88B-48C4-A26A-BB43AF4186BE", + "lastModified": "2008-04-15T22:37:41.910", + "created": "2007-08-23T21:05:57.937", + "titles": [ + { + "title": "HP Scanjet Utilities", + "lang": "en" + } + ] + } + }, + { + "cpe": { + "deprecated": false, + "cpeName": "cpe:2.3:a:hp:secure_web_console:-:*:*:*:*:*:*:*", + "cpeNameId": "01F51BEC-78C7-4910-8328-77B8D69ED767", + "lastModified": "2007-09-14T17:36:49.090", + "created": "2007-08-23T21:05:57.937", + "titles": [ + { + "title": "HP Secure Web Console", + "lang": "en" + } + ] + } + }, + { + "cpe": { + "deprecated": false, + "cpeName": "cpe:2.3:a:hp:sendmail:-:*:*:*:*:*:*:*", + "cpeNameId": "F2CAC9D2-0F87-433C-9D9E-9C99D347D56A", + "lastModified": "2007-09-14T17:36:49.090", + "created": "2007-08-23T21:05:57.937", + "titles": [ + { + "title": "HP sendmail", + "lang": "en" + } + ] + } + }, + { + "cpe": { + "deprecated": true, + "cpeName": "cpe:2.3:o:linux:linux_kernel:2.6.2:*:*:*:*:*:*:*", + "cpeNameId": "DF3171C4-00E8-4B0F-97EB-2F3EC3394A87", + "lastModified": "2021-06-01T14:14:47.707", + "created": "2007-08-23T21:16:59.567", + "titles": [ + { + "title": "Linux Kernel 2.6.2", + "lang": "en" + } + ], + "refs": [ + { + "ref": "https://github.com/torvalds/linux", + "type": "Version" + } + ], + "deprecatedBy": [ + { + "cpeName": "cpe:2.3:o:linux:linux_kernel:2.6.2:-:*:*:*:*:*:*", + "cpeNameId": "1B4C49FC-8606-45D7-94D1-19C5626D69C7" + } + ], + "deprecates": [ + { + "cpeName": "cpe:2.3:o:linux:kernel:2.6.2:*:*:*:*:*:*:*", + "cpeNameId": "B548E49E-BC95-4804-A2C2-D7ACC7F72095" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/cmd/cpe/testdata/test2.golden b/cmd/cpe/testdata/test2.golden new file mode 100644 index 0000000000..01088adc87 --- /dev/null +++ b/cmd/cpe/testdata/test2.golden @@ -0,0 +1 @@ +[[cpe:2.3:a:denkgroot:spina:2.3.5:*:*:*:*:*:*:* Denkgroot Spina 2.3.5 denkgroot spina 2.3.5 false] [cpe:2.3:a:denkgroot:spina:2.3.4:*:*:*:*:*:*:* Denkgroot Spina 2.3.4 denkgroot spina 2.3.4 false]] \ No newline at end of file diff --git a/cmd/cpe/testdata/test2.json b/cmd/cpe/testdata/test2.json new file mode 100644 index 0000000000..0c9334a8c6 --- /dev/null +++ b/cmd/cpe/testdata/test2.json @@ -0,0 +1,60 @@ +{ + "resultsPerPage": 2, + "startIndex": 1110000, + "totalResults": 2, + "format": "NVD_CPE", + "version": "2.0", + "timestamp": "2023-11-07T21:11:14.883", + "products": [ + { + "cpe": { + "deprecated": false, + "cpeName": "cpe:2.3:a:denkgroot:spina:2.3.5:*:*:*:*:*:*:*", + "cpeNameId": "6F467674-732A-4EAB-9728-C3D5CE9950A9", + "lastModified": "2023-07-05T16:44:44.687", + "created": "2023-07-05T13:53:03.087", + "titles": [ + { + "title": "Denkgroot Spina 2.3.5", + "lang": "en" + } + ], + "refs": [ + { + "ref": "https:\/\/huntr.dev\/bounties\/18a74a9d-4a2d-4bf8-ae62-56a909427070\/", + "type": "Advisory" + }, + { + "ref": "https:\/\/github.com\/SpinaCMS\/Spina\/releases", + "type": "Change Log" + } + ] + } + }, + { + "cpe": { + "deprecated": false, + "cpeName": "cpe:2.3:a:denkgroot:spina:2.3.4:*:*:*:*:*:*:*", + "cpeNameId": "048B1A11-455D-4EB5-8198-29FD8DBDEC4D", + "lastModified": "2023-07-05T16:44:44.687", + "created": "2023-07-05T13:53:03.087", + "titles": [ + { + "title": "Denkgroot Spina 2.3.4", + "lang": "en" + } + ], + "refs": [ + { + "ref": "https:\/\/huntr.dev\/bounties\/18a74a9d-4a2d-4bf8-ae62-56a909427070\/", + "type": "Advisory" + }, + { + "ref": "https:\/\/github.com\/SpinaCMS\/Spina\/releases", + "type": "Change Log" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/go.mod b/go.mod index 1c6565c02e..692feaf4c9 100644 --- a/go.mod +++ b/go.mod @@ -78,6 +78,7 @@ require ( github.com/open-policy-agent/opa v0.44.0 github.com/oschwald/geoip2-golang v1.8.0 github.com/osquery/osquery-go v0.0.0-20230603132358-d2e851b3991b + github.com/pandatix/nvdapi v0.6.4 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.13.0 @@ -227,6 +228,7 @@ require ( github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/goreleaser/chglog v0.1.2 // indirect github.com/goreleaser/fileglob v1.2.0 // indirect + github.com/gorilla/schema v1.2.0 // indirect github.com/groob/finalizer v0.0.0-20170707115354-4c2ed49aabda // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index 602ce1cdc3..fcba080f5a 100644 --- a/go.sum +++ b/go.sum @@ -690,6 +690,8 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/schema v1.2.0 h1:YufUaxZYCKGFuAq3c96BOhjgd5nmXiOY9NGzF247Tsc= +github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -977,6 +979,8 @@ github.com/oschwald/maxminddb-golang v1.10.0 h1:Xp1u0ZhqkSuopaKmk1WwHtjF0H9Hd918 github.com/oschwald/maxminddb-golang v1.10.0/go.mod h1:Y2ELenReaLAZ0b400URyGwvYxHV1dLIxBuyOsyYjHK0= github.com/osquery/osquery-go v0.0.0-20230603132358-d2e851b3991b h1:kPna3NDVHKquM7hGLWcztO6eH+NTbTprHfGKrClGJqk= github.com/osquery/osquery-go v0.0.0-20230603132358-d2e851b3991b/go.mod h1:OSR0OKXZZ+mnt08q14OndgHjJJ9/1koA2dDO3jzYr/I= +github.com/pandatix/nvdapi v0.6.4 h1:gix57FcQtOklCUgFrJzJhRblYj+2DN9jxZP6oqtme+A= +github.com/pandatix/nvdapi v0.6.4/go.mod h1:DVYxPq0JRERgYzFmwTMknAtH4kB8v9KG+z40JWFRClk= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= diff --git a/server/vulnerabilities/nvd/cpe_test.go b/server/vulnerabilities/nvd/cpe_test.go index 8dfbb35b77..6cca1492cb 100644 --- a/server/vulnerabilities/nvd/cpe_test.go +++ b/server/vulnerabilities/nvd/cpe_test.go @@ -29,7 +29,7 @@ func TestCPEFromSoftware(t *testing.T) { dbPath := filepath.Join(tempDir, "cpe.sqlite") - err = GenerateCPEDB(dbPath, items) + err = GenerateCPEDB(dbPath, items.Items) require.NoError(t, err) db, err := sqliteDB(dbPath) @@ -56,7 +56,7 @@ func TestCPETranslations(t *testing.T) { dbPath := filepath.Join(tempDir, "cpe.sqlite") - err = GenerateCPEDB(dbPath, items) + err = GenerateCPEDB(dbPath, items.Items) require.NoError(t, err) db, err := sqliteDB(dbPath) @@ -348,7 +348,7 @@ func TestTranslateSoftwareToCPE(t *testing.T) { require.NoError(t, err) dbPath := filepath.Join(tempDir, "cpe.sqlite") - err = GenerateCPEDB(dbPath, items) + err = GenerateCPEDB(dbPath, items.Items) require.NoError(t, err) err = TranslateSoftwareToCPE(context.Background(), ds, tempDir, kitlog.NewNopLogger()) @@ -397,7 +397,7 @@ func TestTranslateSoftwareToCPEIgnoreEmptyVersion(t *testing.T) { require.NoError(t, err) dbPath := filepath.Join(tempDir, "cpe.sqlite") - err = GenerateCPEDB(dbPath, items) + err = GenerateCPEDB(dbPath, items.Items) require.NoError(t, err) err = TranslateSoftwareToCPE(context.Background(), ds, tempDir, kitlog.NewNopLogger()) @@ -444,7 +444,7 @@ func TestLegacyCPEDB(t *testing.T) { dbPath := filepath.Join(tempDir, "cpe.sqlite") - err = GenerateCPEDB(dbPath, items) + err = GenerateCPEDB(dbPath, items.Items) require.NoError(t, err) db, err := sqliteDB(dbPath) diff --git a/server/vulnerabilities/nvd/db.go b/server/vulnerabilities/nvd/db.go index 3e65f529a8..690132171f 100644 --- a/server/vulnerabilities/nvd/db.go +++ b/server/vulnerabilities/nvd/db.go @@ -84,7 +84,7 @@ func generateCPEItem(item cpedict.CPEItem) ([]interface{}, map[string]string, er const batchSize = 800 -func GenerateCPEDB(path string, items *cpedict.CPEList) error { +func GenerateCPEDB(path string, items []cpedict.CPEItem) error { err := os.Remove(path) if err != nil && !errors.Is(err, os.ErrNotExist) { return err @@ -105,7 +105,7 @@ func GenerateCPEDB(path string, items *cpedict.CPEList) error { deprecationsCount := 0 var deprecationsBatch []interface{} - for _, item := range items.Items { + for _, item := range items { cpes, deprecations, err := generateCPEItem(item) if err != nil { return err