Add CentOS parsing+post-processing to reduce false positives in vulnerability processing (#4037)

* Add CentOS parsing and post-processing in fleet

* Add tests and amend SyncCPEDatabase

* Add test for centosPostProcessing

* Changes from PR comments

* Amend software test

* Fix sync test

* Add index to source and vendor

* Use os.MkdirTemp

* Rearrange migrations

* Regenerate test schema

* Add support for testing migrations (#4112)

* Add support for testing migrations

* Rename migration in tests

* Changes suggested in PR

* Go mod tidy

cmd/cpe/generate.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"compress/gzip"
+	"database/sql"
+	"flag"
 	"fmt"
 	"io"
 	"net/http"
@@ -11,6 +13,7 @@ import (
 
 	"github.com/facebookincubator/nvdtools/cpedict"
 	"github.com/fleetdm/fleet/v4/server/vulnerabilities"
+	"github.com/fleetdm/fleet/v4/server/vulnerabilities/vuln_centos"
 )
 
 func panicif(err error) {
@@ -20,11 +23,43 @@ func panicif(err error) {
 }
 
 func main() {
-	fmt.Println("Starting CPE sqlite generation")
+	var (
+		runCentOS bool
+		verbose   bool
+	)
+	flag.BoolVar(&runCentOS, "centos", true, "Sets whether to run the CentOS sqlite generation")
+	flag.BoolVar(&verbose, "verbose", false, "Sets verbose mode")
+	flag.Parse()
+
+	dbPath := cpe()
+
+	fmt.Printf("Sqlite file %s size: %.2f MB\n", dbPath, getSizeMB(dbPath))
+
+	// The CentOS repository data is added to the CPE database.
+	if runCentOS {
+		centos(dbPath, verbose)
+		fmt.Printf("Sqlite file %s size with CentOS data: %.2f MB\n", dbPath, getSizeMB(dbPath))
+	}
+
+	fmt.Println("Compressing DB...")
+	compressedPath, err := compress(dbPath)
+	panicif(err)
+
+	fmt.Printf("Final compressed file %s size: %.2f MB\n", compressedPath, getSizeMB(compressedPath))
+	fmt.Println("Done.")
+}
+
+func getSizeMB(path string) float64 {
+	info, err := os.Stat(path)
+	panicif(err)
+	return float64(info.Size()) / 1024.0 / 1024.0
+}
+
+func cpe() string {
+	fmt.Println("Starting CPE sqlite generation...")
 
 	cwd, err := os.Getwd()
 	panicif(err)
-
 	fmt.Println("CWD:", cwd)
 
 	resp, err := http.Get("https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz")
@@ -34,16 +69,6 @@ func main() {
 	remoteEtag := getSanitizedEtag(resp)
 	fmt.Println("Got ETag:", remoteEtag)
 
-	nvdRelease, err := vulnerabilities.GetLatestNVDRelease(nil)
-	panicif(err)
-
-	if nvdRelease != nil && nvdRelease.Etag == remoteEtag {
-		fmt.Println("No updates. Exiting...")
-		return
-	}
-
-	fmt.Println("Needs updating. Generating...")
-
 	gr, err := gzip.NewReader(resp.Body)
 	panicif(err)
 	defer gr.Close()
@@ -56,25 +81,51 @@ func main() {
 	err = vulnerabilities.GenerateCPEDB(dbPath, cpeDict)
 	panicif(err)
 
-	fmt.Println("Compressing db...")
-	compressedDB, err := os.Create(fmt.Sprintf("%s.gz", dbPath))
-	panicif(err)
-
-	db, err := os.Open(dbPath)
-	panicif(err)
-	w := gzip.NewWriter(compressedDB)
-
-	_, err = io.Copy(w, db)
-	panicif(err)
-	w.Close()
-	compressedDB.Close()
-
 	file, err := os.Create(path.Join(cwd, "etagenv"))
 	panicif(err)
 	file.WriteString(fmt.Sprintf(`ETAG=%s`, remoteEtag))
 	file.Close()
 
-	fmt.Println("Done.")
+	return dbPath
+}
+
+func compress(path string) (string, error) {
+	compressedPath := fmt.Sprintf("%s.gz", path)
+	compressedDB, err := os.Create(compressedPath)
+	if err != nil {
+		return "", err
+	}
+	defer compressedDB.Close()
+
+	db, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer db.Close()
+
+	w := gzip.NewWriter(compressedDB)
+	defer w.Close()
+
+	_, err = io.Copy(w, db)
+	if err != nil {
+		return "", err
+	}
+	return compressedPath, nil
+}
+
+func centos(dbPath string, verbose bool) {
+	fmt.Println("Starting CentOS sqlite generation...")
+
+	db, err := sql.Open("sqlite3", dbPath)
+	panicif(err)
+	defer db.Close()
+
+	pkgs, err := vuln_centos.ParseCentOSRepository(vuln_centos.WithVerbose(verbose))
+	panicif(err)
+
+	fmt.Printf("Storing CVE info for %d CentOS packages...\n", len(pkgs))
+	err = vuln_centos.GenCentOSSqlite(db, pkgs)
+	panicif(err)
 }
 
 func getSanitizedEtag(resp *http.Response) string {