diff --git a/.gitignore b/.gitignore index 5d078b3dad..c7e524795e 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,8 @@ terraform.tfstate* # generated installers fleet-osquery* + +# residual files when running the cpe command +cmd/cpe/etagenv +cmd/cpe/cpe*.sqlite +cmd/cpe/cpe*.sqlite.gz diff --git a/changes/issue-3081-parse-centos-repository b/changes/issue-3081-parse-centos-repository new file mode 100644 index 0000000000..40fc8592c9 --- /dev/null +++ b/changes/issue-3081-parse-centos-repository @@ -0,0 +1 @@ +* Add parsing of the CentOS repository as well as the loading of the results in fleet vulnerability post-processing (to reduce false positives on RPM packages). diff --git a/cmd/cpe/generate.go b/cmd/cpe/generate.go index 9b29b6460c..a0f9cab877 100644 --- a/cmd/cpe/generate.go +++ b/cmd/cpe/generate.go @@ -2,6 +2,8 @@ package main import ( "compress/gzip" + "database/sql" + "flag" "fmt" "io" "net/http" @@ -11,6 +13,7 @@ import ( "github.com/facebookincubator/nvdtools/cpedict" "github.com/fleetdm/fleet/v4/server/vulnerabilities" + "github.com/fleetdm/fleet/v4/server/vulnerabilities/vuln_centos" ) func panicif(err error) { @@ -20,11 +23,43 @@ func panicif(err error) { } func main() { - fmt.Println("Starting CPE sqlite generation") + var ( + runCentOS bool + verbose bool + ) + flag.BoolVar(&runCentOS, "centos", true, "Sets whether to run the CentOS sqlite generation") + flag.BoolVar(&verbose, "verbose", false, "Sets verbose mode") + flag.Parse() + + dbPath := cpe() + + fmt.Printf("Sqlite file %s size: %.2f MB\n", dbPath, getSizeMB(dbPath)) + + // The CentOS repository data is added to the CPE database. + if runCentOS { + centos(dbPath, verbose) + fmt.Printf("Sqlite file %s size with CentOS data: %.2f MB\n", dbPath, getSizeMB(dbPath)) + } + + fmt.Println("Compressing DB...") + compressedPath, err := compress(dbPath) + panicif(err) + + fmt.Printf("Final compressed file %s size: %.2f MB\n", compressedPath, getSizeMB(compressedPath)) + fmt.Println("Done.") +} + +func getSizeMB(path string) float64 { + info, err := os.Stat(path) + panicif(err) + return float64(info.Size()) / 1024.0 / 1024.0 +} + +func cpe() string { + fmt.Println("Starting CPE sqlite generation...") cwd, err := os.Getwd() panicif(err) - fmt.Println("CWD:", cwd) resp, err := http.Get("https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz") @@ -34,16 +69,6 @@ func main() { remoteEtag := getSanitizedEtag(resp) fmt.Println("Got ETag:", remoteEtag) - nvdRelease, err := vulnerabilities.GetLatestNVDRelease(nil) - panicif(err) - - if nvdRelease != nil && nvdRelease.Etag == remoteEtag { - fmt.Println("No updates. Exiting...") - return - } - - fmt.Println("Needs updating. Generating...") - gr, err := gzip.NewReader(resp.Body) panicif(err) defer gr.Close() @@ -56,25 +81,51 @@ func main() { err = vulnerabilities.GenerateCPEDB(dbPath, cpeDict) panicif(err) - fmt.Println("Compressing db...") - compressedDB, err := os.Create(fmt.Sprintf("%s.gz", dbPath)) - panicif(err) - - db, err := os.Open(dbPath) - panicif(err) - w := gzip.NewWriter(compressedDB) - - _, err = io.Copy(w, db) - panicif(err) - w.Close() - compressedDB.Close() - file, err := os.Create(path.Join(cwd, "etagenv")) panicif(err) file.WriteString(fmt.Sprintf(`ETAG=%s`, remoteEtag)) file.Close() - fmt.Println("Done.") + return dbPath +} + +func compress(path string) (string, error) { + compressedPath := fmt.Sprintf("%s.gz", path) + compressedDB, err := os.Create(compressedPath) + if err != nil { + return "", err + } + defer compressedDB.Close() + + db, err := os.Open(path) + if err != nil { + return "", err + } + defer db.Close() + + w := gzip.NewWriter(compressedDB) + defer w.Close() + + _, err = io.Copy(w, db) + if err != nil { + return "", err + } + return compressedPath, nil +} + +func centos(dbPath string, verbose bool) { + fmt.Println("Starting CentOS sqlite generation...") + + db, err := sql.Open("sqlite3", dbPath) + panicif(err) + defer db.Close() + + pkgs, err := vuln_centos.ParseCentOSRepository(vuln_centos.WithVerbose(verbose)) + panicif(err) + + fmt.Printf("Storing CVE info for %d CentOS packages...\n", len(pkgs)) + err = vuln_centos.GenCentOSSqlite(db, pkgs) + panicif(err) } func getSanitizedEtag(resp *http.Response) string { diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index f4b4794e6c..89fb4a88f8 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -722,6 +722,16 @@ func cronVulnerabilities( sentry.CaptureException(err) } + // It's important vulnerabilities.PostProcess runs after ds.CalculateHostsPerSoftware + // because it cleans up any software that's not installed on the fleet (e.g. hosts removal, + // or software being uninstalled on hosts). + if !vulnDisabled { + if err := vulnerabilities.PostProcess(ctx, ds, vulnPath, logger, config); err != nil { + level.Error(logger).Log("msg", "post processing CVEs", "err", err) + sentry.CaptureException(err) + } + } + level.Debug(logger).Log("loop", "done") } } diff --git a/cmd/fleetctl/vulnerability_data_stream.go b/cmd/fleetctl/vulnerability_data_stream.go index b350855712..10f1c3ce06 100644 --- a/cmd/fleetctl/vulnerability_data_stream.go +++ b/cmd/fleetctl/vulnerability_data_stream.go @@ -47,7 +47,7 @@ Downloads (if needed) the data streams that can be used by the Fleet server to p dbPath := path.Join(dir, "cpe.sqlite") client := fleethttp.NewClient() - err = vulnerabilities.SyncCPEDatabase(client, dbPath, config.FleetConfig{}) + err = vulnerabilities.SyncCPEDatabase(client, dbPath) if err != nil { return err } diff --git a/docs/03-Contributing/02-Testing.md b/docs/03-Contributing/02-Testing.md index 7c675429f7..a3ad8e5074 100644 --- a/docs/03-Contributing/02-Testing.md +++ b/docs/03-Contributing/02-Testing.md @@ -226,7 +226,7 @@ To intercept sent emails while running a Fleet development environment, first, i Then, in the "SMTP options" section, enter any email address in the "Sender address" field, set the "SMTP server" to `localhost` on port `1025`, and set "Authentication type" to `None`. Note that you may use any active or inactive sender address. -Visit [locahost:8025](http://localhost:8025) to view Mailhog's admin interface which will display all emails sent using the simulated mail server. +Visit [localhost:8025](http://localhost:8025) to view Mailhog's admin interface which will display all emails sent using the simulated mail server. ## Development database management diff --git a/go.mod b/go.mod index 92b4ae206b..262d43ca1d 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,12 @@ require ( github.com/AbGuthrie/goquery/v2 v2.0.1 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/OneOfOne/xxhash v1.2.8 // indirect + github.com/PuerkitoBio/goquery v1.8.0 // indirect github.com/VividCortex/gohistogram v1.0.0 // indirect github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f github.com/WatchBeam/clock v0.0.0-20170901150240-b08e6b4da7ea + github.com/antchfx/htmlquery v1.2.4 // indirect + github.com/antchfx/xmlquery v1.3.9 // indirect github.com/aws/aws-sdk-go v1.40.34 github.com/beevik/etree v1.1.0 github.com/briandowns/spinner v1.13.0 @@ -29,6 +32,7 @@ require ( github.com/ghodss/yaml v1.0.0 github.com/go-kit/kit v0.9.0 github.com/go-sql-driver/mysql v1.6.0 + github.com/gocolly/colly v1.2.0 github.com/golang-jwt/jwt/v4 v4.0.0 github.com/gomodule/redigo v1.8.5 github.com/google/go-cmp v0.5.6 @@ -45,6 +49,7 @@ require ( github.com/jinzhu/copier v0.3.2 github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/kennygrant/sanitize v1.2.4 // indirect github.com/kevinburke/go-bindata v3.22.0+incompatible github.com/kolide/kit v0.0.0-20180421083548-36eb8dc43916 github.com/kolide/launcher v0.0.0-20180427153757-cb412b945cf7 @@ -68,14 +73,18 @@ require ( github.com/rotisserie/eris v0.5.1 github.com/rs/zerolog v1.20.0 github.com/russellhaering/goxmldsig v1.1.0 + github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 github.com/spf13/cast v1.3.1 github.com/spf13/cobra v1.2.1 github.com/spf13/viper v1.8.1 github.com/stretchr/objx v0.3.0 // indirect github.com/stretchr/testify v1.7.0 + github.com/temoto/robotstxt v1.1.2 // indirect github.com/theupdateframework/go-tuf v0.0.0-20220121203041-e3557e322879 github.com/throttled/throttled/v2 v2.8.0 + github.com/tj/assert v0.0.3 + github.com/ulikunitz/xz v0.5.10 github.com/urfave/cli/v2 v2.3.0 github.com/valyala/fasthttp v1.31.0 golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce diff --git a/go.sum b/go.sum index 01b3e3754a..2112f4f2fb 100644 --- a/go.sum +++ b/go.sum @@ -164,6 +164,8 @@ github.com/ProtonMail/go-mime v0.0.0-20190923161245-9b5a4261663a h1:W6RrgN/sTxg1 github.com/ProtonMail/go-mime v0.0.0-20190923161245-9b5a4261663a/go.mod h1:NYt+V3/4rEeDuaev/zw1zCq8uqVEuPHzDPo3OZrlGJ4= github.com/ProtonMail/gopenpgp/v2 v2.2.2 h1:u2m7xt+CZWj88qK1UUNBoXeJCFJwJCZ/Ff4ymGoxEXs= github.com/ProtonMail/gopenpgp/v2 v2.2.2/go.mod h1:ajUlBGvxMH1UBZnaYO3d1FSVzjiC6kK9XlZYGiDCvpM= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= @@ -189,8 +191,16 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andybalholm/brotli v1.0.2 h1:JKnhI/XQ75uFBTiuzXpzFrUriDPiZjlOSzh6wXogP0E= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494= +github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc= +github.com/antchfx/xmlquery v1.3.9 h1:Y+zyMdiUZ4fasTQTkDb3DflOXP7+obcYEh80SISBmnQ= +github.com/antchfx/xmlquery v1.3.9/go.mod h1:wojC/BxjEkjJt6dPiAqUzoXO5nIMWtxHS8PD8TmN4ks= +github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8= +github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.13.1-0.20200603211036-eac4d0c79a5f h1:33BV5v3u8I6dA2dEoPuXWCsAaHHOJfPtdxZhAMQV4uo= github.com/apache/thrift v0.13.1-0.20200603211036-eac4d0c79a5f/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -436,6 +446,8 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= +github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -682,6 +694,8 @@ github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYb github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kevinburke/go-bindata v3.22.0+incompatible h1:/JmqEhIWQ7GRScV0WjX/0tqBrC5D21ALg0H0U/KZ/ts= github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= @@ -914,6 +928,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc= @@ -990,6 +1006,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/technoweenie/multipartstreamer v1.0.1 h1:XRztA5MXiR1TIRHxH2uNxXxaIkKQDeX7m2XsSOlQEnM= github.com/technoweenie/multipartstreamer v1.0.1/go.mod h1:jNVxdtShOxzAsukZwTSw6MDx5eUJoiEBsSvzDU9uzog= +github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= +github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/theupdateframework/go-tuf v0.0.0-20220121203041-e3557e322879 h1:UeDpdrX16scCvbdgdMsrztZsQLDofld/Zo+WGDe/PBE= github.com/theupdateframework/go-tuf v0.0.0-20220121203041-e3557e322879/go.mod h1:I0Gs4Tev4hYQ5wiNqN8VJ7qS0gw7KOZNQuckC624RmE= github.com/throttled/throttled/v2 v2.8.0 h1:B5VfdM8BE+ClI2Ji238SbNOTWfYcocvuAhgT27lvwrE= @@ -1052,7 +1070,6 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/zclconf/go-cty v1.1.0 h1:uJwc9HiBOCpoKIObTQaLR+tsEXx1HBHnOsOOpcdhZgw= github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c h1:TWQ2UvXPkhPxI2KmApKBOCaV6yD2N4mlvqFQ/DlPtpQ= github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c/go.mod h1:OYYulo9tUqRadRLwB0+LE914sa1ui2yL7OrcU3Q/1XY= @@ -1183,6 +1200,7 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -1190,6 +1208,7 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -1209,6 +1228,7 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211007125505-59d4e928ea9d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= diff --git a/pkg/README.md b/pkg/README.md index be6a15939d..53ca5f0601 100644 --- a/pkg/README.md +++ b/pkg/README.md @@ -1,3 +1,3 @@ # pkg directory -This top-level `pkg` directory contains packages that may be shared between `fleet` and `orbit`. +This top-level `pkg` directory contains packages that may be shared between all `fleet` backend components. diff --git a/pkg/download/download.go b/pkg/download/download.go new file mode 100644 index 0000000000..eb012d0e84 --- /dev/null +++ b/pkg/download/download.go @@ -0,0 +1,70 @@ +// Package download has utilities to download resources from URLs. +package download + +import ( + "compress/bzip2" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/ulikunitz/xz" +) + +// Decompressed downloads and decompresses a file from a URL to a local path. +// +// It supports gz, bz2 and gz compressed files. +func Decompressed(client *http.Client, u url.URL, path string) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + tmpFile, err := ioutil.TempFile("", fmt.Sprintf("%s*", filepath.Base(path))) + if err != nil { + return err + } + defer tmpFile.Close() + + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return err + } + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + var decompressor io.Reader + switch { + case strings.HasSuffix(u.Path, "gz"): + decompressor, err = gzip.NewReader(resp.Body) + if err != nil { + return err + } + case strings.HasSuffix(u.Path, "bz2"): + decompressor = bzip2.NewReader(resp.Body) + case strings.HasSuffix(u.Path, "xz"): + decompressor, err = xz.NewReader(resp.Body) + if err != nil { + return err + } + default: + return fmt.Errorf("unknown extension: %s", u.Path) + } + if _, err := io.Copy(tmpFile, decompressor); err != nil { + return err + } + if err := tmpFile.Close(); err != nil { + return err + } + if err := os.Rename(tmpFile.Name(), path); err != nil { + return err + } + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20220208144831_AddSoftwareReleaseArchVendorColumns.go b/server/datastore/mysql/migrations/tables/20220208144831_AddSoftwareReleaseArchVendorColumns.go new file mode 100644 index 0000000000..be40fee8cb --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20220208144831_AddSoftwareReleaseArchVendorColumns.go @@ -0,0 +1,66 @@ +package tables + +import ( + "database/sql" + "fmt" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20220208144831, Down_20220208144831) +} + +func Up_20220208144831(tx *sql.Tx) error { + // NOTE(lucas): I'm using short lengths for the new varchar columns + // due to constraints on the size of the key added below. Using 255 + // for the three new fields would fail with: + // "Error 1071: Specified key was too long; max key length is 3072 bytes". + // + // We need to use "NOT NULL" because these new columns are to be included in the KEY. + if _, err := tx.Exec("ALTER TABLE software " + + "ADD COLUMN `release` VARCHAR(64) NOT NULL DEFAULT '', " + + "ADD COLUMN vendor VARCHAR(32) NOT NULL DEFAULT '', " + + "ADD COLUMN arch VARCHAR(16) NOT NULL DEFAULT ''"); err != nil { + return errors.Wrap(err, "add new software columns") + } + + // Delete current identifier for a software. + currIndexName, err := indexNameByColumnName(tx, "software", "name") + if err != nil { + return errors.Wrap(err, "fetch current software index") + } + if _, err := tx.Exec(fmt.Sprintf("ALTER TABLE software DROP KEY %s", currIndexName)); err != nil { + return errors.Wrap(err, "add new software columns") + } + + // A software piece was originally identified by name, version and source. + // + // We now add "vendor", "release" and "arch": + // - release is the version of the OS this software was released on (e.g. "30.el7" for a CentOS package). + // - vendor is the supplier of the software (e.g. "CentOS"). + // - arch is the target architecture of the software (e.g. "x86_64"). + if _, err := tx.Exec("ALTER TABLE software ADD UNIQUE KEY (name, version, source, `release`, vendor, arch)"); err != nil { + return errors.Wrap(err, "add new index") + } + + // Remove all software with source rpm_packages, as we will be ingesting them with new osquery + // fields. + // + // Due to foreign keys, the following statement also deletes the corresponding + // entries in `software_cpe` and `software_cve`. + if _, err := tx.Exec("DELETE FROM software WHERE source = 'rpm_packages'"); err != nil { + return errors.Wrap(err, "delete existing software for rpm_packages") + } + + // Adding index to optimize software listing by source and vendor for vulnerability post-processing. + if _, err := tx.Exec("CREATE INDEX software_source_vendor_idx ON software (source, vendor)"); err != nil { + return errors.Wrap(err, "creating source+vendor index") + } + + return nil +} + +func Down_20220208144831(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20220208144831_AddSoftwareReleaseArchVendorColumns_test.go b/server/datastore/mysql/migrations/tables/20220208144831_AddSoftwareReleaseArchVendorColumns_test.go new file mode 100644 index 0000000000..b5fdf7573b --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20220208144831_AddSoftwareReleaseArchVendorColumns_test.go @@ -0,0 +1,35 @@ +package tables + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUp_20220208144831(t *testing.T) { + db := applyUpToPrev(t) + + _, err := db.Exec(`INSERT INTO software (name, version, source) VALUES ("authconfig", "6.2.8", "rpm_packages")`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO software (name, version, source, bundle_identifier) VALUES ("iTerm.app", "3.4.14", "apps", "com.googlecode.iterm2")`) + require.NoError(t, err) + + applyNext(t, db) + + // Check migration removes rpm packages. + row := db.QueryRow(`SELECT COUNT(*) FROM software WHERE source = "rpm_packages"`) + var count int + require.NoError(t, row.Scan(&count)) + require.Zero(t, count) + row = db.QueryRow(`SELECT COUNT(*) FROM software WHERE source = "apps"`) + require.NoError(t, row.Scan(&count)) + require.Equal(t, 1, count) + + // Check we can INSERT software with the new columns empty. + _, err = db.Exec(`INSERT INTO software (name, version, source, bundle_identifier) VALUES ("iCloud.app", "1.0", "apps", "com.apple.CloudKit.ShareBear")`) + require.NoError(t, err) + + // Check we can INSERT software with the new columns set. + _, err = db.Exec(`INSERT INTO software (name, version, source, ` + "`release`" + `, vendor, arch) VALUES ("authconfig", "6.2.8", "rpm_packages", "30.el7", "CentOS", "x86_64")`) + require.NoError(t, err) +} diff --git a/server/datastore/mysql/migrations/tables/migration_test.go b/server/datastore/mysql/migrations/tables/migration_test.go new file mode 100644 index 0000000000..8ef18cfa0d --- /dev/null +++ b/server/datastore/mysql/migrations/tables/migration_test.go @@ -0,0 +1,86 @@ +// Package tables holds fleet table migrations. +// +// Migrations can be tested with tests following the following format: +// +// $ cat 20220208144831_AddSoftwareReleaseArchVendorColumns_test.go +// +// [...] +// func TestUp_20220208144831(t *testing.T) { +// // Apply all migrations up to 20220208144831 (name of test), not included. +// db := applyUpToPrev(t) +// +// // insert testing data, etc. +// +// // The following will apply migration 20220208144831. +// applyNext(t, db) +// +// // insert testing data, verify migration. +// } +package tables + +import ( + "fmt" + "strconv" + "strings" + "testing" + + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/require" +) + +// TODO(lucas): I'm copy pasting some of the mysql functionality methods here +// otherwise we have import cycle errors. +// +// We need to decouple the server/datastore/mysql package, +// it contains both the implementation of the fleet.Datastore and +// MySQL functionality, and MySQL test functionality. +const ( + testUsername = "root" + testPassword = "toor" + testAddress = "localhost:3307" +) + +func newDBConnForTests(t *testing.T) *sqlx.DB { + db, err := sqlx.Open( + "mysql", + fmt.Sprintf("%s:%s@tcp(%s)/?multiStatements=true", testUsername, testPassword, testAddress), + ) + require.NoError(t, err) + _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s; CREATE DATABASE %s; USE %s;", t.Name(), t.Name(), t.Name())) + require.NoError(t, err) + return db +} + +func getMigrationVersion(t *testing.T) int64 { + v, err := strconv.Atoi(strings.TrimPrefix(t.Name(), "TestUp_")) + require.NoError(t, err) + return int64(v) +} + +// applyUpToPrev will allocate a testing DB connection and apply +// migrations up to, not including, the migration specified in the test name. +// +// It returns the database connection to perform additional queries and migrations. +func applyUpToPrev(t *testing.T) *sqlx.DB { + db := newDBConnForTests(t) + v := getMigrationVersion(t) + for { + current, err := MigrationClient.GetDBVersion(db.DB) + require.NoError(t, err) + next, err := MigrationClient.Migrations.Next(current) + require.NoError(t, err) + if next.Version == v { + return db + } + applyNext(t, db) + } +} + +// applyNext performs the next migration in the chain. +func applyNext(t *testing.T, db *sqlx.DB) { + // gooseNoDir is the value to not parse local files and instead use + // the migrations that were added manually via Add(). + const gooseNoDir = "" + err := MigrationClient.UpByOne(db.DB, gooseNoDir) + require.NoError(t, err) +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 7f5cb07499..755a2b8e52 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -317,9 +317,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=122 DEFAULT CHARSET=utf8mb4; +) ENGINE=InnoDB AUTO_INCREMENT=123 DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `network_interfaces` ( @@ -519,9 +519,13 @@ CREATE TABLE `software` ( `version` varchar(255) NOT NULL DEFAULT '', `source` varchar(64) NOT NULL, `bundle_identifier` varchar(255) DEFAULT '', + `release` varchar(64) NOT NULL DEFAULT '', + `vendor` varchar(32) NOT NULL DEFAULT '', + `arch` varchar(16) NOT NULL DEFAULT '', PRIMARY KEY (`id`), - UNIQUE KEY `idx_name_version` (`name`,`version`,`source`), - KEY `software_listing_idx` (`name`,`id`) + UNIQUE KEY `name` (`name`,`version`,`source`,`release`,`vendor`,`arch`), + KEY `software_listing_idx` (`name`,`id`), + KEY `software_source_vendor_idx` (`source`,`vendor`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index eb98ce37f4..ce9a9488d0 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -19,6 +19,10 @@ const ( maxSoftwareVersionLen = 255 maxSoftwareSourceLen = 64 maxSoftwareBundleIdentifierLen = 255 + + maxSoftwareReleaseLen = 64 + maxSoftwareVendorLen = 32 + maxSoftwareArchLen = 16 ) func truncateString(str string, length int) string { @@ -29,16 +33,36 @@ func truncateString(str string, length int) string { } func softwareToUniqueString(s fleet.Software) string { - return strings.Join([]string{s.Name, s.Version, s.Source, s.BundleIdentifier}, "\u0000") + ss := []string{s.Name, s.Version, s.Source, s.BundleIdentifier} + // Release, Vendor and Arch fields were added on a migration, + // thus we only include them in the string if at least one of them is defined. + if s.Release != "" || s.Vendor != "" || s.Arch != "" { + ss = append(ss, s.Release, s.Vendor, s.Arch) + } + return strings.Join(ss, "\u0000") } func uniqueStringToSoftware(s string) fleet.Software { parts := strings.Split(s, "\u0000") + + // Release, Vendor and Arch fields were added on a migration, + // If one of them is defined, then they are included in the string. + var release, vendor, arch string + if len(parts) > 4 { + release = truncateString(parts[4], maxSoftwareReleaseLen) + vendor = truncateString(parts[5], maxSoftwareVendorLen) + arch = truncateString(parts[6], maxSoftwareArchLen) + } + return fleet.Software{ Name: truncateString(parts[0], maxSoftwareNameLen), Version: truncateString(parts[1], maxSoftwareVersionLen), Source: truncateString(parts[2], maxSoftwareSourceLen), BundleIdentifier: truncateString(parts[3], maxSoftwareBundleIdentifierLen), + + Release: release, + Vendor: vendor, + Arch: arch, } } @@ -151,8 +175,10 @@ func getOrGenerateSoftwareIdDB(ctx context.Context, tx sqlx.ExtContext, s fleet. var existingId []int64 if err := sqlx.SelectContext(ctx, tx, &existingId, - `SELECT id FROM software WHERE name = ? AND version = ? AND source = ? AND bundle_identifier = ?`, - s.Name, s.Version, s.Source, s.BundleIdentifier, + "SELECT id FROM software "+ + "WHERE name = ? AND version = ? AND source = ? AND `release` = ? AND "+ + "vendor = ? AND arch = ? AND bundle_identifier = ?", + s.Name, s.Version, s.Source, s.Release, s.Vendor, s.Arch, s.BundleIdentifier, ); err != nil { return 0, ctxerr.Wrap(ctx, err, "get software") } @@ -161,9 +187,11 @@ func getOrGenerateSoftwareIdDB(ctx context.Context, tx sqlx.ExtContext, s fleet. } result, err := tx.ExecContext(ctx, - `INSERT INTO software (name, version, source, bundle_identifier) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE bundle_identifier=VALUES(bundle_identifier)`, - s.Name, s.Version, s.Source, s.BundleIdentifier, + "INSERT INTO software "+ + "(name, version, source, `release`, vendor, arch, bundle_identifier) "+ + "VALUES (?, ?, ?, ?, ?, ?, ?) "+ + "ON DUPLICATE KEY UPDATE bundle_identifier=VALUES(bundle_identifier)", + s.Name, s.Version, s.Source, s.Release, s.Vendor, s.Arch, s.BundleIdentifier, ) if err != nil { return 0, ctxerr.Wrap(ctx, err, "insert software") @@ -503,6 +531,59 @@ func (ds *Datastore) CountSoftware(ctx context.Context, opt fleet.SoftwareListOp return countSoftwareDB(ctx, ds.reader, nil, opt) } +// ListVulnerableSoftwareBySource lists all the vulnerable software that matches the given source. +func (ds *Datastore) ListVulnerableSoftwareBySource(ctx context.Context, source string) ([]fleet.SoftwareWithCPE, error) { + var softwareCVEs []struct { + fleet.Software + CPE uint `db:"cpe_id"` + CVEs string `db:"cves"` + } + if err := sqlx.SelectContext(ctx, ds.reader, &softwareCVEs, ` + SELECT s.*, scv.cpe_id, GROUP_CONCAT(scv.cve SEPARATOR ',') as cves + FROM software s + JOIN software_cpe scp ON (s.id=scp.software_id) + JOIN software_cve scv ON (scp.id=scv.cpe_id) + WHERE s.source = ? + GROUP BY scv.cpe_id + `, source); err != nil { + return nil, ctxerr.Wrapf(ctx, err, "listing vulnerable software by source") + } + software := make([]fleet.SoftwareWithCPE, 0, len(softwareCVEs)) + for _, sc := range softwareCVEs { + for _, cve := range strings.Split(sc.CVEs, ",") { + sc.Software.Vulnerabilities = append(sc.Software.Vulnerabilities, fleet.SoftwareCVE{ + CVE: cve, + DetailsLink: fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cve), + }) + } + software = append(software, fleet.SoftwareWithCPE{ + Software: sc.Software, + CPEID: sc.CPE, + }) + } + return software, nil +} + +// DeleteVulnerabilitiesByCPECVE deletes the given list of vulnerabilities identified by CPE+CVE. +func (ds *Datastore) DeleteVulnerabilitiesByCPECVE(ctx context.Context, vulnerabilities []fleet.SoftwareVulnerability) error { + if len(vulnerabilities) == 0 { + return nil + } + + sql := fmt.Sprintf( + `DELETE FROM software_cve WHERE (cpe_id, cve) IN (%s)`, + strings.TrimSuffix(strings.Repeat("(?,?),", len(vulnerabilities)), ","), + ) + var args []interface{} + for _, vulnerability := range vulnerabilities { + args = append(args, vulnerability.CPEID, vulnerability.CVE) + } + if _, err := ds.writer.ExecContext(ctx, sql, args...); err != nil { + return ctxerr.Wrapf(ctx, err, "deleting vulnerable software") + } + return nil +} + func (ds *Datastore) SoftwareByID(ctx context.Context, id uint) (*fleet.Software, error) { software := fleet.Software{} err := sqlx.GetContext(ctx, ds.reader, &software, `SELECT * FROM software WHERE id=?`, id) @@ -545,6 +626,9 @@ func (ds *Datastore) SoftwareByID(ctx context.Context, id uint) (*fleet.Software // CalculateHostsPerSoftware calculates the number of hosts having each // software installed and stores that information in the software_host_counts // table. +// +// After aggregation, it cleans up unused software (e.g. software installed +// on removed hosts, software uninstalled on hosts, etc.) func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error { resetStmt := ` UPDATE software_host_counts @@ -614,8 +698,6 @@ func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt ti return ctxerr.Wrap(ctx, err, "iterate over host_software counts") } - // delete any unused software from the software table (any that - // isn't in that list with a host count > 0). cleanupStmt := ` DELETE FROM software @@ -630,7 +712,6 @@ func (ds *Datastore) CalculateHostsPerSoftware(ctx context.Context, updatedAt ti if _, err := ds.writer.ExecContext(ctx, cleanupStmt); err != nil { return ctxerr.Wrap(ctx, err, "delete unused software") } - return nil } diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index 63146909b4..795f286fe8 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -33,6 +33,8 @@ func TestSoftware(t *testing.T) { {"LoadSupportsTonsOfCVEs", testSoftwareLoadSupportsTonsOfCVEs}, {"List", testSoftwareList}, {"CalculateHostsPerSoftware", testSoftwareCalculateHostsPerSoftware}, + {"ListVulnerableSoftwareBySource", testListVulnerableSoftwareBySource}, + {"DeleteVulnerabilitiesByCPECVE", testDeleteVulnerabilitiesByCPECVE}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -426,7 +428,15 @@ func testSoftwareList(t *testing.T, ds *Datastore) { t.Run("paginates", func(t *testing.T) { software := listSoftwareCheckCount(t, ds, 1, 4, fleet.SoftwareListOptions{ListOptions: fleet.ListOptions{Page: 1, PerPage: 1, OrderKey: "version"}}, true) - expected := []fleet.Software{foo003} + require.Len(t, software, 1) + var expected []fleet.Software + // Both foo003 and bar003 have the same version, thus we check which one the database picked + // for the second page. + if software[0].Name == "foo" { + expected = []fleet.Software{foo003} + } else { + expected = []fleet.Software{bar003} + } test.ElementsMatchSkipID(t, software, expected) }) @@ -626,3 +636,145 @@ func testSoftwareCalculateHostsPerSoftware(t *testing.T, ds *Datastore) { } cmpNameVersionCount(want, allSw) } + +func insertVulnSoftwareForTest(t *testing.T, ds *Datastore) { + host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) + host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) + + software1 := []fleet.Software{ + { + Name: "foo.rpm", Version: "0.0.1", Source: "rpm_packages", GenerateCPE: "cpe_foo_rpm", + }, + { + Name: "foo.chrome", Version: "0.0.3", Source: "chrome_extensions", GenerateCPE: "cpe_foo_chrome", + }, + } + software2 := []fleet.Software{ + { + Name: "foo.chrome", Version: "v0.0.2", Source: "chrome_extensions", GenerateCPE: "cpe_foo_chrome2", + }, + { + Name: "foo.chrome", Version: "0.0.3", Source: "chrome_extensions", GenerateCPE: "cpe_foo_chrome_3", + Vulnerabilities: fleet.VulnerabilitiesSlice{ + {CVE: "cve-123-456-789", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-123-456-789"}, + }, + }, + { + Name: "bar.rpm", Version: "0.0.3", Source: "rpm_packages", GenerateCPE: "cpe_bar_rpm", + Vulnerabilities: fleet.VulnerabilitiesSlice{ + {CVE: "cve-321-432-543", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-321-432-543"}, + {CVE: "cve-333-444-555", DetailsLink: "https://nvd.nist.gov/vuln/detail/cve-333-444-555"}, + }, + }, + } + + require.NoError(t, ds.UpdateHostSoftware(context.Background(), host1.ID, software1)) + require.NoError(t, ds.UpdateHostSoftware(context.Background(), host2.ID, software2)) + require.NoError(t, ds.LoadHostSoftware(context.Background(), host1)) + require.NoError(t, ds.LoadHostSoftware(context.Background(), host2)) + + sort.Slice(host1.Software, func(i, j int) bool { + return host1.Software[i].Name+host1.Software[i].Version < host1.Software[j].Name+host1.Software[j].Version + }) + sort.Slice(host2.Software, func(i, j int) bool { + return host2.Software[i].Name+host2.Software[i].Version < host2.Software[j].Name+host2.Software[j].Version + }) + + require.NoError(t, ds.AddCPEForSoftware(context.Background(), host1.Software[0], "cpe_foo_chrome")) + require.NoError(t, ds.AddCPEForSoftware(context.Background(), host1.Software[1], "cpe_foo_rpm")) + + require.NoError(t, ds.AddCPEForSoftware(context.Background(), host2.Software[0], "cpe_bar_rpm")) + require.NoError(t, ds.AddCPEForSoftware(context.Background(), host2.Software[1], "cpe_foo_chrome_3")) + require.NoError(t, ds.AddCPEForSoftware(context.Background(), host2.Software[2], "cpe_foo_chrome_2")) + + _, err := ds.InsertCVEForCPE(context.Background(), "cve-123-456-789", []string{"cpe_foo_chrome_3"}) + require.NoError(t, err) + + _, err = ds.InsertCVEForCPE(context.Background(), "cve-321-432-543", []string{"cpe_bar_rpm"}) + require.NoError(t, err) + _, err = ds.InsertCVEForCPE(context.Background(), "cve-333-444-555", []string{"cpe_bar_rpm"}) + require.NoError(t, err) +} + +func testListVulnerableSoftwareBySource(t *testing.T, ds *Datastore) { + ctx := context.Background() + + insertVulnSoftwareForTest(t, ds) + + vulnerable, err := ds.ListVulnerableSoftwareBySource(ctx, "apps") + require.NoError(t, err) + require.Empty(t, vulnerable) + + vulnerable, err = ds.ListVulnerableSoftwareBySource(ctx, "rpm_packages") + require.NoError(t, err) + require.Len(t, vulnerable, 1) + require.Equal(t, vulnerable[0].Name, "bar.rpm") + require.Len(t, vulnerable[0].Vulnerabilities, 2) + sort.Slice(vulnerable[0].Vulnerabilities, func(i, j int) bool { + return vulnerable[0].Vulnerabilities[i].CVE < vulnerable[0].Vulnerabilities[j].CVE + }) + require.Equal(t, "cve-321-432-543", vulnerable[0].Vulnerabilities[0].CVE) + require.Equal(t, "cve-333-444-555", vulnerable[0].Vulnerabilities[1].CVE) +} + +func testDeleteVulnerabilitiesByCPECVE(t *testing.T, ds *Datastore) { + ctx := context.Background() + + err := ds.DeleteVulnerabilitiesByCPECVE(ctx, nil) + require.NoError(t, err) + + insertVulnSoftwareForTest(t, ds) + + err = ds.DeleteVulnerabilitiesByCPECVE(ctx, []fleet.SoftwareVulnerability{ + { + CPEID: 999, // unknown CPE + CVE: "cve-333-444-555", + }, + }) + require.NoError(t, err) + + software, err := ds.ListVulnerableSoftwareBySource(ctx, "rpm_packages") + require.NoError(t, err) + + require.Len(t, software, 1) + barRPM := software[0] + require.Len(t, barRPM.Vulnerabilities, 2) + + err = ds.DeleteVulnerabilitiesByCPECVE(ctx, []fleet.SoftwareVulnerability{ + { + CPEID: barRPM.CPEID, + CVE: "unknown-cve", + }, + }) + require.NoError(t, err) + + err = ds.DeleteVulnerabilitiesByCPECVE(ctx, []fleet.SoftwareVulnerability{ + { + CPEID: barRPM.CPEID, + CVE: "cve-333-444-555", + }, + }) + require.NoError(t, err) + + software, err = ds.ListVulnerableSoftwareBySource(ctx, "rpm_packages") + require.NoError(t, err) + require.Len(t, software, 1) + barRPM = software[0] + require.Len(t, barRPM.Vulnerabilities, 1) + + err = ds.DeleteVulnerabilitiesByCPECVE(ctx, []fleet.SoftwareVulnerability{ + { + CPEID: barRPM.CPEID, + CVE: "cve-321-432-543", + }, + }) + require.NoError(t, err) + + software, err = ds.ListVulnerableSoftwareBySource(ctx, "rpm_packages") + require.NoError(t, err) + require.Len(t, software, 0) + + software, err = ds.ListVulnerableSoftwareBySource(ctx, "chrome_extensions") + require.NoError(t, err) + require.Len(t, software, 1) +} diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index a87d3c01e4..afde10cabf 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "time" ) @@ -330,6 +331,12 @@ type Datastore interface { AllCPEs(ctx context.Context) ([]string, error) InsertCVEForCPE(ctx context.Context, cve string, cpes []string) (int64, error) SoftwareByID(ctx context.Context, id uint) (*Software, error) + // CalculateHostsPerSoftware calculates the number of hosts having each + // software installed and stores that information in the software_host_counts + // table. + // + // After aggregation, it cleans up unused software (e.g. software installed + // on removed hosts, software uninstalled on hosts, etc.) CalculateHostsPerSoftware(ctx context.Context, updatedAt time.Time) error HostsByCPEs(ctx context.Context, cpes []string) ([]*CPEHost, error) @@ -374,6 +381,10 @@ type Datastore interface { ListSoftware(ctx context.Context, opt SoftwareListOptions) ([]Software, error) CountSoftware(ctx context.Context, opt SoftwareListOptions) (int, error) + // ListVulnerableSoftwareBySource lists all the vulnerable software that matches the given source. + ListVulnerableSoftwareBySource(ctx context.Context, source string) ([]SoftwareWithCPE, error) + // DeleteVulnerabilities deletes the given list of vulnerabilities identified by CPE+CVE. + DeleteVulnerabilitiesByCPECVE(ctx context.Context, vulnerabilities []SoftwareVulnerability) error /////////////////////////////////////////////////////////////////////////////// // Team Policies @@ -549,6 +560,26 @@ const ( UnknownMigrations ) +// SoftwareVulnerability identifies a vulnerability on a specific software (CPE). +type SoftwareVulnerability struct { + // CPEID is the ID of the software CPE in the system. + CPEID uint + CVE string +} + +// String implements fmt.Stringer. +func (sv SoftwareVulnerability) String() string { + return fmt.Sprintf("{%d,%s}", sv.CPEID, sv.CVE) +} + +// SoftwareWithCPE holds a software piece alongside its CPE ID. +type SoftwareWithCPE struct { + // Software holds the software data. + Software + // CPEID is the ID of the software CPE in the system. + CPEID uint +} + // NotFoundError is returned when the datastore resource cannot be found. type NotFoundError interface { error diff --git a/server/fleet/software.go b/server/fleet/software.go index 68ceb92daf..45bce98277 100644 --- a/server/fleet/software.go +++ b/server/fleet/software.go @@ -19,6 +19,14 @@ type Software struct { // Source is the source of the data (osquery table name). Source string `json:"source" db:"source"` + // Release is the version of the OS this software was released on + // (e.g. "30.el7" for a CentOS package). + Release string `json:"release,omitempty" db:"release"` + // Vendor is the supplier of the software (e.g. "CentOS"). + Vendor string `json:"vendor,omitempty" db:"vendor"` + // Arch is the architecture of the software (e.g. "x86_64"). + Arch string `json:"arch,omitempty" db:"arch"` + // GenerateCPE is the CPE23 string that corresponds to the current software GenerateCPE string `json:"generated_cpe" db:"generated_cpe"` // Vulnerabilities lists all the found CVEs for the CPE diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 98c4c6e358..109c3e8563 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -312,6 +312,10 @@ type ListSoftwareFunc func(ctx context.Context, opt fleet.SoftwareListOptions) ( type CountSoftwareFunc func(ctx context.Context, opt fleet.SoftwareListOptions) (int, error) +type ListVulnerableSoftwareBySourceFunc func(ctx context.Context, source string) ([]fleet.SoftwareWithCPE, error) + +type DeleteVulnerabilitiesByCPECVEFunc func(ctx context.Context, vulnerabilities []fleet.SoftwareVulnerability) error + type NewTeamPolicyFunc func(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) type ListTeamPoliciesFunc func(ctx context.Context, teamID uint) ([]*fleet.Policy, error) @@ -825,6 +829,12 @@ type DataStore struct { CountSoftwareFunc CountSoftwareFunc CountSoftwareFuncInvoked bool + ListVulnerableSoftwareBySourceFunc ListVulnerableSoftwareBySourceFunc + ListVulnerableSoftwareBySourceFuncInvoked bool + + DeleteVulnerabilitiesByCPECVEFunc DeleteVulnerabilitiesByCPECVEFunc + DeleteVulnerabilitiesByCPECVEFuncInvoked bool + NewTeamPolicyFunc NewTeamPolicyFunc NewTeamPolicyFuncInvoked bool @@ -1669,6 +1679,16 @@ func (s *DataStore) CountSoftware(ctx context.Context, opt fleet.SoftwareListOpt return s.CountSoftwareFunc(ctx, opt) } +func (s *DataStore) ListVulnerableSoftwareBySource(ctx context.Context, source string) ([]fleet.SoftwareWithCPE, error) { + s.ListVulnerableSoftwareBySourceFuncInvoked = true + return s.ListVulnerableSoftwareBySourceFunc(ctx, source) +} + +func (s *DataStore) DeleteVulnerabilitiesByCPECVE(ctx context.Context, vulnerabilities []fleet.SoftwareVulnerability) error { + s.DeleteVulnerabilitiesByCPECVEFuncInvoked = true + return s.DeleteVulnerabilitiesByCPECVEFunc(ctx, vulnerabilities) +} + func (s *DataStore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) { s.NewTeamPolicyFuncInvoked = true return s.NewTeamPolicyFunc(ctx, teamID, authorID, args) diff --git a/server/ptr/ptr.go b/server/ptr/ptr.go index ea61035c30..c9ca71f97b 100644 --- a/server/ptr/ptr.go +++ b/server/ptr/ptr.go @@ -11,6 +11,8 @@ func String(x string) *string { return &x } +// StringValueOrZero returns the string value. +// Returns empty string if x is nil. func StringValueOrZero(x *string) string { if x == nil { return "" diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index a4dace831c..d11f4acca9 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -84,6 +84,10 @@ func (s *integrationTestSuite) TearDownTest() { _, err = s.ds.DeleteGlobalPolicies(ctx, globalPolicyIDs) require.NoError(t, err) } + + // CalculateHostsPerSoftware performs a cleanup. + err = s.ds.CalculateHostsPerSoftware(ctx, time.Now()) + require.NoError(t, err) } func TestIntegrations(t *testing.T) { @@ -1742,13 +1746,15 @@ func (s *integrationTestSuite) TestScheduledQueries() { // batch-delete by id, 3 ids, only one exists var delBatchResp deleteQueriesResponse s.DoJSON("POST", "/api/v1/fleet/queries/delete", map[string]interface{}{ - "ids": []uint{query.ID, query2.ID, query3.ID}}, http.StatusOK, &delBatchResp) + "ids": []uint{query.ID, query2.ID, query3.ID}, + }, http.StatusOK, &delBatchResp) assert.Equal(t, uint(1), delBatchResp.Deleted) // batch-delete by id, none exist delBatchResp.Deleted = 0 s.DoJSON("POST", "/api/v1/fleet/queries/delete", map[string]interface{}{ - "ids": []uint{query.ID, query2.ID, query3.ID}}, http.StatusNotFound, &delBatchResp) + "ids": []uint{query.ID, query2.ID, query3.ID}, + }, http.StatusNotFound, &delBatchResp) assert.Equal(t, uint(0), delBatchResp.Deleted) } @@ -2615,7 +2621,8 @@ func (s *integrationTestSuite) TestQuerySpecs() { // delete all queries created var delBatchResp deleteQueriesResponse s.DoJSON("POST", "/api/v1/fleet/queries/delete", map[string]interface{}{ - "ids": []uint{q1ID, q2ID, q3ID}}, http.StatusOK, &delBatchResp) + "ids": []uint{q1ID, q2ID, q3ID}, + }, http.StatusOK, &delBatchResp) assert.Equal(t, uint(3), delBatchResp.Deleted) } diff --git a/server/service/osquery_utils/queries.go b/server/service/osquery_utils/queries.go index fc6ade2d52..a704beecfc 100644 --- a/server/service/osquery_utils/queries.go +++ b/server/service/osquery_utils/queries.go @@ -395,56 +395,80 @@ SELECT name AS name, version AS version, 'Package (deb)' AS type, - 'deb_packages' AS source + 'deb_packages' AS source, + '' AS release, + '' AS vendor, + '' AS arch FROM deb_packages UNION SELECT package AS name, version AS version, 'Package (Portage)' AS type, - 'portage_packages' AS source + 'portage_packages' AS source, + '' AS release, + '' AS vendor, + '' AS arch FROM portage_packages UNION SELECT name AS name, version AS version, 'Package (RPM)' AS type, - 'rpm_packages' AS source + 'rpm_packages' AS source, + release AS release, + vendor AS vendor, + arch AS arch FROM rpm_packages UNION SELECT name AS name, version AS version, 'Package (NPM)' AS type, - 'npm_packages' AS source + 'npm_packages' AS source, + '' AS release, + '' AS vendor, + '' AS arch FROM npm_packages UNION SELECT name AS name, version AS version, 'Browser plugin (Chrome)' AS type, - 'chrome_extensions' AS source + 'chrome_extensions' AS source, + '' AS release, + '' AS vendor, + '' AS arch FROM cached_users CROSS JOIN chrome_extensions USING (uid) UNION SELECT name AS name, version AS version, 'Browser plugin (Firefox)' AS type, - 'firefox_addons' AS source + 'firefox_addons' AS source, + '' AS release, + '' AS vendor, + '' AS arch FROM cached_users CROSS JOIN firefox_addons USING (uid) UNION SELECT name AS name, version AS version, 'Package (Atom)' AS type, - 'atom_packages' AS source + 'atom_packages' AS source, + '' AS release, + '' AS vendor, + '' AS arch FROM cached_users CROSS JOIN atom_packages USING (uid) UNION SELECT name AS name, version AS version, 'Package (Python)' AS type, - 'python_packages' AS source + 'python_packages' AS source, + '' AS release, + '' AS vendor, + '' AS arch FROM python_packages; `, Platforms: fleet.HostLinuxOSs, @@ -649,11 +673,16 @@ func directIngestSoftware(ctx context.Context, logger log.Logger, host *fleet.Ho ) continue } + s := fleet.Software{ Name: name, Version: version, Source: source, BundleIdentifier: bundleIdentifier, + + Release: row["release"], + Vendor: row["vendor"], + Arch: row["arch"], } software = append(software, s) } diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go index 397d383cf8..a885d3f125 100644 --- a/server/service/service_osquery.go +++ b/server/service/service_osquery.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "regexp" "strconv" "strings" "sync" @@ -17,7 +18,6 @@ import ( "github.com/fleetdm/fleet/v4/server/contexts/logging" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/ptr" - "github.com/fleetdm/fleet/v4/server/pubsub" "github.com/fleetdm/fleet/v4/server/service/osquery_utils" "github.com/go-kit/kit/log" @@ -761,6 +761,8 @@ func (svc *Service) directIngestDetailQuery(ctx context.Context, host *fleet.Hos return false, nil } +var noSuchTableRegexp = regexp.MustCompile(`^no such table: \S+$`) + func (svc *Service) SubmitDistributedQueryResults( ctx context.Context, results fleet.OsqueryDistributedQueryResults, @@ -787,6 +789,9 @@ func (svc *Service) SubmitDistributedQueryResults( // osquery docs say any nonzero (string) value for status indicates a query error status, ok := statuses[query] failed := ok && status != fleet.StatusOK + if failed && messages[query] != "" && !noSuchTableRegexp.MatchString(messages[query]) { + level.Debug(svc.logger).Log("query", query, "message", messages[query]) + } var err error switch { case strings.HasPrefix(query, hostDetailQueryPrefix): diff --git a/server/vulnerabilities/centos.go b/server/vulnerabilities/centos.go new file mode 100644 index 0000000000..20e242a81e --- /dev/null +++ b/server/vulnerabilities/centos.go @@ -0,0 +1,92 @@ +package vulnerabilities + +import ( + "context" + "database/sql" + "fmt" + + "github.com/fleetdm/fleet/v4/server/config" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/vulnerabilities/vuln_centos" + kitlog "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" +) + +// centosPostProcessing performs processing over the list of vulnerable rpm packages +// and removes the vulnerabilities where the CVEs are known to be fixed. +func centosPostProcessing( + ctx context.Context, + ds fleet.Datastore, + db *sql.DB, + logger kitlog.Logger, + config config.FleetConfig, +) error { + centOSPkgs, err := vuln_centos.LoadCentOSFixedCVEs(ctx, db, logger) + if err != nil { + return fmt.Errorf("failed to fetch CentOS packages: %w", err) + } + level.Info(logger).Log("centosPackages", len(centOSPkgs)) + if len(centOSPkgs) == 0 { + return nil + } + + rpmVulnerable, err := ds.ListVulnerableSoftwareBySource(ctx, "rpm_packages") + if err != nil { + return fmt.Errorf("failed to list vulnerable software: %w", err) + } + level.Info(logger).Log("vulnerable rpm_packages", len(rpmVulnerable)) + if len(rpmVulnerable) == 0 { + return nil + } + + var fixedCVEs []fleet.SoftwareVulnerability + var softwareCount int + for _, software := range rpmVulnerable { + if software.Vendor != "CentOS" { + continue + } + pkgFixedCVEs, ok := centOSPkgs[vuln_centos.CentOSPkg{ + Name: software.Name, + Version: software.Version, + Release: software.Release, + Arch: software.Arch, + }] + if !ok { + continue + } + var cves []string + for _, vulnerability := range software.Vulnerabilities { + if _, ok := pkgFixedCVEs[vulnerability.CVE]; ok { + cves = append(cves, vulnerability.CVE) + fixedCVEs = append(fixedCVEs, fleet.SoftwareVulnerability{ + CPEID: software.CPEID, + CVE: vulnerability.CVE, + }) + } + } + if len(cves) > 0 { + softwareCount++ + + level.Debug(logger).Log( + "msg", "fixedCVEs", + "software", fmt.Sprintf( + "%s-%s-%s.%s", + software.Name, software.Version, software.Release, software.Arch, + ), + "softwareCPE", software.CPEID, + "cves", fmt.Sprintf("%v", cves), + ) + } + } + + level.Info(logger).Log( + "msg", "CentOS fixed CVEs", + "fixedCVEsCount", len(fixedCVEs), + "distinctSoftwareCount", softwareCount, + ) + + if err := ds.DeleteVulnerabilitiesByCPECVE(ctx, fixedCVEs); err != nil { + return fmt.Errorf("failed to delete fixed vulnerabilities: %w", err) + } + return nil +} diff --git a/server/vulnerabilities/centos_test.go b/server/vulnerabilities/centos_test.go new file mode 100644 index 0000000000..759a228beb --- /dev/null +++ b/server/vulnerabilities/centos_test.go @@ -0,0 +1,152 @@ +package vulnerabilities + +import ( + "context" + "database/sql" + "testing" + + "github.com/fleetdm/fleet/v4/server/config" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/mock" + "github.com/fleetdm/fleet/v4/server/vulnerabilities/vuln_centos" + "github.com/go-kit/kit/log" + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/require" +) + +func TestCentOSPostProcessing(t *testing.T) { + ctx := context.Background() + ds := new(mock.Store) + + db, err := sql.Open("sqlite3", ":memory:") + require.NoError(t, err) + + pkgs := make(vuln_centos.CentOSPkgSet) + authConfigPkg := vuln_centos.CentOSPkg{ + Name: "authconfig", + Version: "6.2.8", + Release: "30.el7", + Arch: "x86_64", + } + pkgs.Add(authConfigPkg, "CVE-2017-7488") + sqlitePkg := vuln_centos.CentOSPkg{ + Name: "sqlite", + Version: "3.7.17", + Release: "8.el7_7", + Arch: "x86_64", + } + pkgs.Add(sqlitePkg, "CVE-2015-3415", "CVE-2015-3416", "CVE-2015-3414") + + err = vuln_centos.GenCentOSSqlite(db, pkgs) + require.NoError(t, err) + + vulnSoftware := []fleet.SoftwareWithCPE{ + { + Software: fleet.Software{ + Name: "authconfig", + Version: "6.2.8", + Release: "30.el7", + Arch: "x86_64", + Vendor: "CentOS", + Vulnerabilities: fleet.VulnerabilitiesSlice{ + { + CVE: "CVE-2017-7488", + }, + }, + }, + CPEID: 1, + }, + { + Software: fleet.Software{ + Name: "sqlite", + Version: "3.7.17", + Release: "8.el7_7", + Arch: "x86_64", + Vendor: "CentOS", + Vulnerabilities: fleet.VulnerabilitiesSlice{ + { + CVE: "CVE-2015-3415", + }, + { + CVE: "CVE-2015-3416", + }, + { + CVE: "CVE-2022-9999", + }, + }, + }, + CPEID: 2, + }, + { + Software: fleet.Software{ + Name: "ghostscript", + Version: "9.25", + Release: "5.el7", + Arch: "x86_64", + Vendor: "CentOS", + Vulnerabilities: fleet.VulnerabilitiesSlice{ + { + CVE: "CVE-2019-3835", + }, + }, + }, + CPEID: 3, + }, + { + Software: fleet.Software{ + Name: "gnutls", + Version: "3.3.29", + Release: "9.el7", + Arch: "x86_64", + Vendor: "", + Vulnerabilities: fleet.VulnerabilitiesSlice{ + { + CVE: "CVE-8888-9999", + }, + }, + }, + CPEID: 4, + }, + } + + ds.ListVulnerableSoftwareBySourceFunc = func(ctx context.Context, source string) ([]fleet.SoftwareWithCPE, error) { + return vulnSoftware, nil + } + + ds.DeleteVulnerabilitiesByCPECVEFunc = func(ctx context.Context, vulnerabilities []fleet.SoftwareVulnerability) error { + require.Equal(t, []fleet.SoftwareVulnerability{ + { + CPEID: 1, + CVE: "CVE-2017-7488", + }, + { + CPEID: 2, + CVE: "CVE-2015-3415", + }, + { + CPEID: 2, + CVE: "CVE-2015-3416", + }, + }, vulnerabilities) + return nil + } + + err = centosPostProcessing(ctx, ds, db, log.NewNopLogger(), config.FleetConfig{}) + require.NoError(t, err) + + require.True(t, ds.ListVulnerableSoftwareBySourceFuncInvoked) + require.True(t, ds.DeleteVulnerabilitiesByCPECVEFuncInvoked) +} + +func TestCentOSPostProcessingNoPkgs(t *testing.T) { + ctx := context.Background() + ds := new(mock.Store) + ds.ListVulnerableSoftwareBySourceFunc = func(ctx context.Context, source string) ([]fleet.SoftwareWithCPE, error) { + t.Error("this method shouldn't be called if there are no pkgs in the CentOS table") + return nil, nil + } + db, err := sql.Open("sqlite3", ":memory:") + require.NoError(t, err) + err = centosPostProcessing(ctx, ds, db, log.NewNopLogger(), config.FleetConfig{}) + require.Error(t, err) +} diff --git a/server/vulnerabilities/cpe.go b/server/vulnerabilities/cpe.go index d07a32405d..dd3228059d 100644 --- a/server/vulnerabilities/cpe.go +++ b/server/vulnerabilities/cpe.go @@ -1,18 +1,18 @@ package vulnerabilities import ( - "compress/gzip" "context" "errors" "fmt" - "io" "net/http" + "net/url" "os" "path" "regexp" "strings" "time" + "github.com/fleetdm/fleet/v4/pkg/download" "github.com/fleetdm/fleet/v4/pkg/fleethttp" "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" @@ -69,22 +69,39 @@ func GetLatestNVDRelease(client *http.Client) (*NVDRelease, error) { }, nil } +type syncOpts struct { + url string +} + +type CPESyncOption func(*syncOpts) + +func WithCPEURL(url string) CPESyncOption { + return func(o *syncOpts) { + o.url = url + } +} + +// SyncCPEDatabase (by default) downloads the CPE database from the +// latest release of github.com/fleetdm/nvd to the given dbPath. +// An alternative URL can be set via the WithCPEURL option. +// +// It won't sync the database at dbPath has an mtime that happened after the +// available database release date. func SyncCPEDatabase( client *http.Client, dbPath string, - config config.FleetConfig, + opts ...CPESyncOption, ) error { - if config.Vulnerabilities.DisableDataSync { - return nil + var o syncOpts + for _, fn := range opts { + fn(&o) } - url := config.Vulnerabilities.CPEDatabaseURL - if url == "" { + if o.url == "" { nvdRelease, err := GetLatestNVDRelease(client) if err != nil { return err } - stat, err := os.Stat(dbPath) if err != nil { if !errors.Is(err, os.ErrNotExist) { @@ -93,34 +110,14 @@ func SyncCPEDatabase( } else if !nvdRelease.CreatedAt.After(stat.ModTime()) { return nil } - url = nvdRelease.CPEURL + o.url = nvdRelease.CPEURL } - req, err := http.NewRequest(http.MethodGet, url, nil) + u, err := url.Parse(o.url) if err != nil { return err } - - resp, err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - gr, err := gzip.NewReader(resp.Body) - if err != nil { - return err - } - defer gr.Close() - - dbFile, err := os.Create(dbPath) - if err != nil { - return err - } - defer dbFile.Close() - - _, err = io.Copy(dbFile, gr) - if err != nil { + if err := download.Decompressed(client, *u, dbPath); err != nil { return err } @@ -227,9 +224,11 @@ func TranslateSoftwareToCPE( ) error { dbPath := path.Join(vulnPath, "cpe.sqlite") - client := fleethttp.NewClient() - if err := SyncCPEDatabase(client, dbPath, config); err != nil { - return ctxerr.Wrap(ctx, err, "sync cpe db") + if !config.Vulnerabilities.DisableDataSync { + client := fleethttp.NewClient() + if err := SyncCPEDatabase(client, dbPath, WithCPEURL(config.Vulnerabilities.CPEDatabaseURL)); err != nil { + return ctxerr.Wrap(ctx, err, "sync cpe db") + } } iterator, err := ds.AllSoftwareWithoutCPEIterator(ctx) diff --git a/server/vulnerabilities/cpe_test.go b/server/vulnerabilities/cpe_test.go index 61581ba764..2cbf2f2b9d 100644 --- a/server/vulnerabilities/cpe_test.go +++ b/server/vulnerabilities/cpe_test.go @@ -75,7 +75,7 @@ func TestSyncCPEDatabase(t *testing.T) { } // first time, db doesn't exist, so it downloads - err = SyncCPEDatabase(client, dbPath, config.FleetConfig{}) + err = SyncCPEDatabase(client, dbPath) require.NoError(t, err) db, err := sqliteDB(dbPath) @@ -107,7 +107,7 @@ func TestSyncCPEDatabase(t *testing.T) { require.NoError(t, err) // then it will download - err = SyncCPEDatabase(client, dbPath, config.FleetConfig{}) + err = SyncCPEDatabase(client, dbPath) require.NoError(t, err) // let's register the mtime for the db @@ -128,9 +128,8 @@ func TestSyncCPEDatabase(t *testing.T) { time.Sleep(2 * time.Second) // let's check it doesn't download because it's new enough - err = SyncCPEDatabase(client, dbPath, config.FleetConfig{}) + err = SyncCPEDatabase(client, dbPath) require.NoError(t, err) - stat, err = os.Stat(dbPath) require.NoError(t, err) require.Equal(t, mtime, stat.ModTime()) @@ -226,27 +225,10 @@ func TestSyncsCPEFromURL(t *testing.T) { dbPath := path.Join(tempDir, "cpe.sqlite") err := SyncCPEDatabase( - client, dbPath, config.FleetConfig{Vulnerabilities: config.VulnerabilitiesConfig{CPEDatabaseURL: ts.URL}}) + client, dbPath, WithCPEURL(ts.URL+"/hello-world.gz")) require.NoError(t, err) stored, err := ioutil.ReadFile(dbPath) require.NoError(t, err) assert.Equal(t, "Hello world!", string(stored)) } - -func TestSyncsCPESkipsIfDisableSync(t *testing.T) { - client := fleethttp.NewClient() - tempDir := t.TempDir() - dbPath := path.Join(tempDir, "cpe.sqlite") - - fleetConfig := config.FleetConfig{ - Vulnerabilities: config.VulnerabilitiesConfig{ - DisableDataSync: true, - }, - } - err := SyncCPEDatabase(client, dbPath, fleetConfig) - require.NoError(t, err) - - _, err = os.Stat(dbPath) - require.ErrorIs(t, err, os.ErrNotExist) -} diff --git a/server/vulnerabilities/cve.go b/server/vulnerabilities/cve.go index e54a3d7b4a..8c7635f022 100644 --- a/server/vulnerabilities/cve.go +++ b/server/vulnerabilities/cve.go @@ -2,9 +2,11 @@ package vulnerabilities import ( "context" + "database/sql" "fmt" "net/url" "os" + "path" "path/filepath" "regexp" "runtime" @@ -140,7 +142,7 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, } cache := cvefeed.NewCache(dict).SetRequireVersion(true).SetMaxSize(-1) // This index consumes too much RAM - //cache.Idx = cvefeed.NewIndex(dict) + // cache.Idx = cvefeed.NewIndex(dict) cpeCh := make(chan *wfn.Attributes) collectVulns := recentVulns != nil @@ -243,3 +245,25 @@ func checkCVEs(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, wg.Wait() return nil } + +// PostProcess performs additional processing over the results of +// the main vulnerability processing run (TranslateSoftwareToCPE+TranslateCPEToCVE). +func PostProcess( + ctx context.Context, + ds fleet.Datastore, + vulnPath string, + logger kitlog.Logger, + config config.FleetConfig, +) error { + dbPath := path.Join(vulnPath, "cpe.sqlite") + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + return fmt.Errorf("failed to open cpe database: %w", err) + } + defer db.Close() + + if err := centosPostProcessing(ctx, ds, db, logger, config); err != nil { + return err + } + return nil +} diff --git a/server/vulnerabilities/vuln_centos/centos.go b/server/vulnerabilities/vuln_centos/centos.go new file mode 100644 index 0000000000..72619b7d7e --- /dev/null +++ b/server/vulnerabilities/vuln_centos/centos.go @@ -0,0 +1,451 @@ +// Package vuln_centos contains a ParseCentOSRepository method to parse the CentOS repository +// to look out for CentOS releases that patch CVEs. It parses the changelogs from the metadata. +// +// It also contains a LoadCentOSFixedCVEs to load the results of the parsing. +// +// Both the parsing and loading of results use sqlite3 as backend storage. +package vuln_centos + +import ( + "context" + "database/sql" + "encoding/xml" + "errors" + "fmt" + "io/fs" + "io/ioutil" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "regexp" + "sort" + "strings" + + "github.com/fleetdm/fleet/v4/pkg/download" + "github.com/fleetdm/fleet/v4/pkg/fleethttp" + kitlog "github.com/go-kit/kit/log" + "github.com/gocolly/colly" + _ "github.com/mattn/go-sqlite3" +) + +// CentOSPkg holds data to identify a CentOS package. +type CentOSPkg struct { + Name string + Version string + Release string + Arch string +} + +// String implements fmt.Stringer. +func (p CentOSPkg) String() string { + return p.Name + "-" + p.Version + "-" + p.Release + "." + p.Arch +} + +// FixedCVESet is a set of fixed CVEs. +type FixedCVESet map[string]struct{} + +// CentOSPkgSet is a set of CentOS packages and their fixed CVEs. +type CentOSPkgSet map[CentOSPkg]FixedCVESet + +// Add adds the given package and CVE/s to the set. +func (p CentOSPkgSet) Add(pkg CentOSPkg, fixedCVEs ...string) { + s := p[pkg] + if s == nil { + s = make(FixedCVESet) + } + for _, fixedCVE := range fixedCVEs { + s[fixedCVE] = struct{}{} + } + p[pkg] = s +} + +const centOSPkgsCVEsTable = "centos_pkgs_fixed_cves" + +// LoadCentOSFixedCVEs loads the CentOS packages with known fixed CVEs from the given sqlite3 db. +func LoadCentOSFixedCVEs(ctx context.Context, db *sql.DB, logger kitlog.Logger) (CentOSPkgSet, error) { + rows, err := db.QueryContext(ctx, fmt.Sprintf(`SELECT name, version, release, arch, cves FROM %s`, centOSPkgsCVEsTable)) + if err != nil { + return nil, fmt.Errorf("failed to fetch packages: %w", err) + } + defer rows.Close() + + pkgs := make(CentOSPkgSet) + for rows.Next() { + var pkg CentOSPkg + var cves string + if err := rows.Scan(&pkg.Name, &pkg.Version, &pkg.Release, &pkg.Arch, &cves); err != nil { + return nil, err + } + for _, cve := range strings.Split(cves, ",") { + pkgs.Add(pkg, "CVE-"+cve) + } + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("failed to traverse packages: %w", err) + } + return pkgs, nil +} + +type centOSOpts struct { + noCrawl bool + verbose bool + localDir string + root string +} + +type CentOSOption func(*centOSOpts) + +func WithLocalDir(dir string) CentOSOption { + return func(o *centOSOpts) { + o.localDir = dir + } +} + +func NoCrawl() CentOSOption { + return func(o *centOSOpts) { + o.noCrawl = true + } +} + +func WithVerbose(v bool) CentOSOption { + return func(o *centOSOpts) { + o.verbose = v + } +} + +func WithRoot(root string) CentOSOption { + return func(o *centOSOpts) { + o.root = root + } +} + +const ( + repositoryDomain = "mirror.centos.org" + repositoryURL = "http://" + repositoryDomain + defaultRoot = "/centos/" +) + +var ( + // Only parse the repository metadata for CentOS 6, 7 and 8. + // + // CentOS 6 maintenance updates ended in 2020-11-30, but we will still + // fetch metadata for CentOS 6 because it's considered as of 2022-02-02 + // a "recent" release. + // + // See https://en.wikipedia.org/wiki/CentOS#CentOS_releases. + recentCentOSPathRegex = regexp.MustCompile(`/centos/[678]\S*`) + // nonReleasePathRegex is used to skip non-package centos directories/files. + nonReleasePathRegex = regexp.MustCompile(`/centos/[^0-9]`) +) + +// ParseCentOSRepository performs the following operations: +// - Crawls the CentOS repository website. To find all the sqlite3 files with +// the packages metadata. +// - Processes all the found sqlite3 files to find all fixed CVEs in each package version. +// It parses the changelogs for each package release and looks for the "CVE-" string. +// +// It writes progress messages to stdout. +func ParseCentOSRepository(opts ...CentOSOption) (CentOSPkgSet, error) { + var opts_ centOSOpts + for _, fn := range opts { + fn(&opts_) + } + + if opts_.localDir == "" && opts_.noCrawl { + return nil, errors.New("invalid options: if no crawl is set, local dir must be set") + } + + if opts_.localDir == "" { + localDir, err := os.MkdirTemp("", "centos*") + if err != nil { + return nil, err + } + opts_.localDir = localDir + } + + fmt.Printf("Using local directory: %s\n", opts_.localDir) + if !opts_.noCrawl { + if err := crawl(opts_.root, opts_.localDir, opts_.verbose); err != nil { + return nil, err + } + } + + pkgs, err := parse(opts_.localDir) + if err != nil { + return nil, err + } + if opts_.verbose { + for pkg, cves := range pkgs { + var cveList []string + for cve := range cves { + cveList = append(cveList, cve) + } + if opts_.verbose { + fmt.Printf("%s: %v\n", pkg, cveList) + } + } + } + + return pkgs, nil +} + +func crawl(root string, localDir string, verbose bool) error { + fmt.Println("Crawling CentOS repository...") + c := colly.NewCollector() + + if err := os.MkdirAll(localDir, 0o755); err != nil { + return err + } + + var repoMDs []url.URL + c.OnHTML("#indexlist .indexcolname a[href]", func(e *colly.HTMLElement) { + href := e.Attr("href") + // Skip going to parent directory. + if strings.HasPrefix(root, href) { + return + } + if nonReleasePathRegex.MatchString(path.Join(e.Request.URL.Path, href)) { + return + } + if !recentCentOSPathRegex.MatchString(path.Join(e.Request.URL.Path, href)) { + if verbose { + fmt.Printf("Ignoring old release: %s\n", path.Join(e.Request.URL.Path, href)) + } + return + } + if href == "repomd.xml" { + u := *e.Request.URL + u.Path = path.Join(u.Path, href) + repoMDs = append(repoMDs, u) + if verbose { + fmt.Printf("%s\n", u.Path) + } + return + } + if !strings.Contains(href, "/") { + return + } + e.Request.Visit(href) + }) + + c.AllowedDomains = append(c.AllowedDomains, repositoryDomain) + + if root == "" { + root = defaultRoot + } + if err := c.Visit(repositoryURL + root); err != nil { + return err + } + + for _, u := range repoMDs { + if err := processRepoMD(u, localDir, verbose); err != nil { + return err + } + } + + return nil +} + +type dbs struct { + primary, other string +} + +func parse(localDir string) (CentOSPkgSet, error) { + fmt.Println("Processing sqlite files...") + + dbPaths := make(map[string]dbs) + if err := filepath.WalkDir(localDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if !strings.HasSuffix(path, ".sqlite") { + return nil + } + dbp := dbPaths[filepath.Dir(path)] + if strings.HasSuffix(path, "-primary.sqlite") { + dbp.primary = path + } else if strings.HasSuffix(path, "-other.sqlite") { + dbp.other = path + } + dbPaths[filepath.Dir(path)] = dbp + return nil + }); err != nil { + return nil, err + } + + allPkgs := make(CentOSPkgSet) + for _, db := range dbPaths { + pkgs, err := processSqlites(db) + if err != nil { + return nil, err + } + for pkg, cves := range pkgs { + for cve := range cves { + allPkgs.Add(pkg, cve) + } + } + } + + return allPkgs, nil +} + +func processRepoMD(mdURL url.URL, localDir string, verbose bool) error { + resp, err := http.Get(mdURL.String()) + if err != nil { + return err + } + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + type location struct { + Href string `xml:"href,attr"` + } + type repoDataItem struct { + Type string `xml:"type,attr"` + Location location `xml:"location"` + } + type repoMetadata struct { + XMLName xml.Name `xml:"repomd"` + Datas []repoDataItem `xml:"data"` + } + var md repoMetadata + if err := xml.Unmarshal(b, &md); err != nil { + return err + } + for _, data := range md.Datas { + if data.Type != "primary_db" && data.Type != "other_db" { + continue + } + sqliteURL := mdURL + sqliteURL.Path = strings.TrimSuffix(sqliteURL.Path, "repomd.xml") + strings.TrimPrefix(data.Location.Href, "repodata/") + if verbose { + fmt.Printf("%s\n", sqliteURL.Path) + } + filePath := filePathfromURL(localDir, sqliteURL) + _, err := os.Stat(filePath) + switch { + case err == nil: + // File already exists, nothing to do. + case errors.Is(err, os.ErrNotExist): + if err := download.Decompressed(fleethttp.NewClient(), sqliteURL, filePath); err != nil { + return err + } + default: + return err + } + } + return nil +} + +func filePathfromURL(dir string, url url.URL) string { + filePath := filepath.Join(dir, url.Path) + filePath = strings.TrimSuffix(filePath, ".bz2") + filePath = strings.TrimSuffix(filePath, ".xz") + filePath = strings.TrimSuffix(filePath, ".gz") + return filePath +} + +func processSqlites(dbPaths dbs) (CentOSPkgSet, error) { + db, err := sql.Open("sqlite3", dbPaths.primary) + if err != nil { + return nil, err + } + defer db.Close() + + if _, err := db.Exec(fmt.Sprintf("ATTACH DATABASE '%s' as other;", dbPaths.other)); err != nil { + return nil, err + } + + db.SetMaxOpenConns(1) + rows, err := db.Query(`SELECT + p.name, p.version, p.release, p.arch, c.changelog + FROM packages p + JOIN other.changelog c ON (p.pkgKey=c.pkgKey) + WHERE c.changelog LIKE '%CVE-%-%';`) + if err != nil { + return nil, err + } + defer rows.Close() + + pkgs := make(CentOSPkgSet) + for rows.Next() { + var p CentOSPkg + var changelog string + if err := rows.Scan(&p.Name, &p.Version, &p.Release, &p.Arch, &changelog); err != nil { + return nil, err + } + cves := parseCVEs(changelog) + for _, cve := range cves { + pkgs.Add(p, cve) + } + } + if err := rows.Err(); err != nil { + return nil, err + } + return pkgs, nil +} + +var cveRegex = regexp.MustCompile(`CVE\-[0-9]+\-[0-9]+`) + +func parseCVEs(changelog string) []string { + return cveRegex.FindAllString(changelog, -1) +} + +// GenCentOSSqlite will store the CentOS package set in the given sqlite handle. +func GenCentOSSqlite(db *sql.DB, pkgs CentOSPkgSet) error { + if err := createTable(db); err != nil { + return err + } + type pkgWithCVEs struct { + pkg CentOSPkg + cves string + } + var pkgsWithCVEs []pkgWithCVEs + for pkg, cves := range pkgs { + var cveList []string + for cve := range cves { + cveList = append(cveList, strings.TrimPrefix(cve, "CVE-")) + } + sort.Slice(cveList, func(i, j int) bool { + return cveList[i] < cveList[j] + }) + pkgsWithCVEs = append(pkgsWithCVEs, pkgWithCVEs{ + pkg: pkg, + cves: strings.Join(cveList, ","), + }) + } + for _, pkgWithCVEs := range pkgsWithCVEs { + if _, err := db.Exec( + fmt.Sprintf("REPLACE INTO %s (name, version, release, arch, cves) VALUES (?, ?, ?, ?, ?)", centOSPkgsCVEsTable), + pkgWithCVEs.pkg.Name, + pkgWithCVEs.pkg.Version, + pkgWithCVEs.pkg.Release, + pkgWithCVEs.pkg.Arch, + pkgWithCVEs.cves, + ); err != nil { + return err + } + } + return nil +} + +func createTable(db *sql.DB) error { + _, err := db.Exec(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( + name TEXT, + version TEXT, + release TEXT, + arch TEXT, + cves TEXT, + + UNIQUE (name, version, release, arch) + );`, centOSPkgsCVEsTable)) + return err +} diff --git a/server/vulnerabilities/vuln_centos/centos_test.go b/server/vulnerabilities/vuln_centos/centos_test.go new file mode 100644 index 0000000000..137c413660 --- /dev/null +++ b/server/vulnerabilities/vuln_centos/centos_test.go @@ -0,0 +1,94 @@ +package vuln_centos + +import ( + "context" + "database/sql" + "os" + "testing" + + "github.com/go-kit/kit/log" + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/require" +) + +func TestCentOSPkgSetAdd(t *testing.T) { + pkgSet := make(CentOSPkgSet) + authConfig := CentOSPkg{ + Name: "authconfig", + Version: "6.2.8", + Release: "30.el7", + Arch: "x86_64", + } + cve1 := "CVE-2017-7488" + pkgSet.Add(authConfig, cve1) + + cve2 := "CVE-2017-7489" + pkgSet.Add(authConfig, cve2) + + curl := CentOSPkg{ + Name: "curl", + Version: "4.2", + Release: "30.el7", + Arch: "x86_64", + } + cve3 := "CVE-2017-7490" + pkgSet.Add(curl, cve1) + pkgSet.Add(curl, cve3) + + require.Len(t, pkgSet, 2) + + require.Len(t, pkgSet[authConfig], 2) + require.Contains(t, pkgSet[authConfig], cve1) + require.Contains(t, pkgSet[authConfig], cve2) + require.NotContains(t, pkgSet[authConfig], cve3) + + require.Len(t, pkgSet[curl], 2) + require.Contains(t, pkgSet[curl], cve1) + require.NotContains(t, pkgSet[curl], cve2) + require.Contains(t, pkgSet[curl], cve3) +} + +func TestParseCentOSRepository(t *testing.T) { + if os.Getenv("NETWORK_TEST") == "" { + t.Skip("set environment variable NETWORK_TEST=1 to run") + } + + // Parse a subset of the CentOS repository. + pkgs, err := ParseCentOSRepository(WithRoot("/centos/7/os/x86_64/repodata/")) + require.NoError(t, err) + + db, err := sql.Open("sqlite3", ":memory:") + require.NoError(t, err) + t.Cleanup(func() { + db.Close() + }) + + err = GenCentOSSqlite(db, pkgs) + require.NoError(t, err) + + pkgSet, err := LoadCentOSFixedCVEs(context.Background(), db, log.NewNopLogger()) + require.NoError(t, err) + + // Shouldn't get _lower_ than what was reported during the development of this test (2221), + // as these are already published releases. + require.GreaterOrEqual(t, len(pkgSet), 2221) + for pkg, cveSet := range pkgSet { + require.NotEmpty(t, pkg.Name) + require.NotEmpty(t, pkg.Version) + require.NotEmpty(t, pkg.Release) + require.NotEmpty(t, pkg.Arch) + require.NotEmpty(t, cveSet) + } + + // Check a known vulnerability fixed on a CentOS release. + authConfig := CentOSPkg{ + Name: "authconfig", + Version: "6.2.8", + Release: "30.el7", + Arch: "x86_64", + } + cve := "CVE-2017-7488" + require.Contains(t, pkgSet, authConfig) + require.Len(t, pkgSet[authConfig], 1) + require.Contains(t, pkgSet[authConfig], cve) +}