Discharge7443/fleet
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

validate generate-cve.yml outputs (#26752)

Browse Source 
https://github.com/fleetdm/fleet/issues/21300

- [x] Added/updated automated tests
- [x] A detailed QA plan exists on the associated ticket (if it isn't
there, work with the product group's QA engineer to add it)
- [x] Manual QA for all new/changed functionality
This commit is contained in:
Konstantin Sykulev
2025-03-12 14:49:47 -05:00
committed by GitHub
parent 997adcebe0
commit bd2b2bcd3b
7 changed files with 216 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/vulnerabilities/goval_dictionary/analyzer.go
+4 -3
View File
@@ -5,6 +5,7 @@ import (
"database/sql"
"errors"
"fmt"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/oval"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/utils"
@@ -34,7 +35,7 @@ func Analyze(
if !platform.IsGovalDictionarySupported() {
return nil, ErrUnsupportedPlatform
}
db, err := loadDb(platform, vulnPath)
db, err := LoadDb(platform, vulnPath)
if err != nil {
return nil, err
}
@@ -117,8 +118,8 @@ func Analyze(
return inserted, nil
}
// loadDb returns the latest goval_dictionary database for the given platform.
func loadDb(platform oval.Platform, vulnPath string) (*Database, error) {
// LoadDb returns the latest goval_dictionary database for the given platform.
func LoadDb(platform oval.Platform, vulnPath string) (*Database, error) {
if !platform.IsGovalDictionarySupported() {
return nil, fmt.Errorf("platform %q not supported", platform)
}
server/vulnerabilities/goval_dictionary/database.go
+23 -5
View File
@@ -3,12 +3,13 @@ package goval_dictionary
import (
"database/sql"
"fmt"
"strings"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/oval"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/utils"
kitlog "github.com/go-kit/log"
"github.com/go-kit/log/level"
"strings"
)
func NewDB(db *sql.DB, platform oval.Platform) *Database {
@@ -20,13 +21,30 @@ type Database struct {
platform oval.Platform
}
const baseSearchStmt = `SELECT packages.version, cves.cve_id
FROM packages join definitions on definitions.id = packages.definition_id
JOIN advisories ON advisories.definition_id = definitions.id JOIN cves ON cves.advisory_id = advisories.id`
func (db Database) Verfiy() error {
searchStmt := fmt.Sprintf("%s LIMIT 1", baseSearchStmt)
affectedSoftwareRows, err := db.sqlite.Query(searchStmt)
if err != nil {
return fmt.Errorf("could not query database: %w", err)
}
defer affectedSoftwareRows.Close()
if affectedSoftwareRows.Err() != nil {
return affectedSoftwareRows.Err()
}
return nil
}
// Eval evaluates the current goval_dictionary database against an OS version and a list of installed software,
// returns all software vulnerabilities found. Logs on any errors so we return as many vulnerabilities as we can.
func (db Database) Eval(software []fleet.Software, logger kitlog.Logger) []fleet.SoftwareVulnerability {
searchStmt := `SELECT packages.version, cves.cve_id
FROM packages join definitions on definitions.id = packages.definition_id
JOIN advisories ON advisories.definition_id = definitions.id JOIN cves ON cves.advisory_id = advisories.id
WHERE packages.name = ? AND packages.arch = ? ORDER BY cve_id, version`
searchStmt := fmt.Sprintf("%s WHERE packages.name = ? AND packages.arch = ? ORDER BY cve_id, version", baseSearchStmt)
vulnerabilities := make([]fleet.SoftwareVulnerability, 0)
for _, swItem := range software {
server/vulnerabilities/goval_dictionary/database_test.go
+29 -1
View File
@@ -2,13 +2,37 @@ package goval_dictionary
import (
"database/sql"
"testing"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/oval"
kitlog "github.com/go-kit/log"
"github.com/stretchr/testify/require"
"testing"
)
func TestVerify(t *testing.T) {
sqlite, err := sql.Open("sqlite3", ":memory:")
if err != nil {
t.Fatal(err)
}
// notice how we are missing the packages.version column
dbSetupQueries := []string{
"CREATE TABLE packages (name TEXT NOT NULL, arch TEXT NOT NULL, definition_id INTEGER NOT NULL)",
"CREATE TABLE definitions (id INTEGER NOT NULL PRIMARY KEY)",
"CREATE TABLE advisories (id INTEGER NOT NULL PRIMARY KEY, definition_id INTEGER NOT NULL)",
"CREATE TABLE cves (cve_id TEXT NOT NULL, advisory_id INTEGER NOT NULL)",
}
for _, query := range dbSetupQueries {
if _, err := sqlite.Exec(query); err != nil {
t.Fatal(err)
}
}
db := NewDB(sqlite, oval.NewPlatform("amzn", "Amazon Linux 2.0.0"))
t.Run("Verify alerts of error", func(t *testing.T) {
require.Error(t, db.Verfiy())
})
}
func TestDatabase(t *testing.T) {
// build minimal slice of goval-dictionary sqlite schema
sqlite, err := sql.Open("sqlite3", ":memory:")
@@ -90,4 +114,8 @@ func TestDatabase(t *testing.T) {
require.Equal(t, uint(235), vulns[2].SoftwareID)
require.Equal(t, uint(235), vulns[3].SoftwareID)
})
t.Run("Verify returns no errors", func(t *testing.T) {
require.NoError(t, db.Verfiy())
})
}
server/vulnerabilities/nvd/sync.go
+47 -10
View File
@@ -185,21 +185,13 @@ func DownloadCISAKnownExploitsFeed(vulnPath string) error {
return nil
}
// LoadCVEMeta loads the cvss scores, epss scores, and known exploits from the previously downloaded feeds and saves
// them to the database.
func LoadCVEMeta(ctx context.Context, logger log.Logger, vulnPath string, ds fleet.Datastore) error {
if !license.IsPremium(ctx) {
level.Info(logger).Log("msg", "skipping cve_meta parsing due to license check")
return nil
}
func CVEMetaFromNVDFeedFiles(metaMap map[string]fleet.CVEMeta, vulnPath string, logger log.Logger) error {
// load cvss scores
files, err := getNVDCVEFeedFiles(vulnPath)
if err != nil {
return fmt.Errorf("get nvd cve feeds: %w", err)
}
metaMap := make(map[string]fleet.CVEMeta)
for _, file := range files {
// Load json files one at a time. Attempting to load them all uses too much memory, > 1 GB.
@@ -236,6 +228,10 @@ func LoadCVEMeta(ctx context.Context, logger log.Logger, vulnPath string, ds fle
}
}
return nil
}
func CVEMetaFromEPSSFeedFiles(metaMap map[string]fleet.CVEMeta, vulnPath string, logger log.Logger) error {
// load epss scores
path := filepath.Join(vulnPath, strings.TrimSuffix(epssFilename, ".gz"))
@@ -254,8 +250,12 @@ func LoadCVEMeta(ctx context.Context, logger log.Logger, vulnPath string, ds fle
metaMap[epssScore.CVE] = score
}
return nil
}
func CVEMetaFromCISAFeedFiles(metaMap map[string]fleet.CVEMeta, vulnPath string, logger log.Logger) error {
// load known exploits
path = filepath.Join(vulnPath, cisaKnownExploitsFilename)
path := filepath.Join(vulnPath, cisaKnownExploitsFilename)
b, err := os.ReadFile(path)
if err != nil {
return err
@@ -283,6 +283,43 @@ func LoadCVEMeta(ctx context.Context, logger log.Logger, vulnPath string, ds fle
metaMap[cve] = meta
}
return nil
}
func CVEMetaFromFiles(vulnPath string, logger log.Logger) (map[string]fleet.CVEMeta, error) {
metaMap := make(map[string]fleet.CVEMeta)
err := CVEMetaFromNVDFeedFiles(metaMap, vulnPath, logger)
if err != nil {
return nil, fmt.Errorf("nvd meta: %w", err)
}
err = CVEMetaFromEPSSFeedFiles(metaMap, vulnPath, logger)
if err != nil {
return nil, fmt.Errorf("epss meta: %w", err)
}
err = CVEMetaFromCISAFeedFiles(metaMap, vulnPath, logger)
if err != nil {
return nil, fmt.Errorf("cisa meta: %w", err)
}
return metaMap, nil
}
// LoadCVEMeta loads the cvss scores, epss scores, and known exploits from the previously downloaded feeds and saves
// them to the database.
func LoadCVEMeta(ctx context.Context, logger log.Logger, vulnPath string, ds fleet.Datastore) error {
if !license.IsPremium(ctx) {
level.Info(logger).Log("msg", "skipping cve_meta parsing due to license check")
return nil
}
metaMap, err := CVEMetaFromFiles(vulnPath, logger)
if err != nil {
return err
}
if len(metaMap) == 0 {
return nil
}
server/vulnerabilities/oval/oval_platform.go
+18 -10
View File
@@ -12,12 +12,21 @@ import (
type Platform string
// OvalFilePrefix is the file prefix used when saving an OVAL artifact.
const OvalFilePrefix = "fleet_oval"
const GovalDictionaryFilePrefix = "fleet_goval_dictionary"
const (
OvalFilePrefix = "fleet_oval"
GovalDictionaryFilePrefix = "fleet_goval_dictionary"
)
// SupportedSoftwareSources are the software sources for which we are using OVAL or goval-dictionary for vulnerability detection.
var SupportedSoftwareSources = []string{"deb_packages", "rpm_packages"}
var SupportedGovalPlatforms = []string{
"amzn_01",
"amzn_02",
"amzn_2022",
"amzn_2023",
}
// getMajorMinorVer returns the major and minor version of an 'os_version'.
// ex: 'Ubuntu 20.4.0' => '(20, 04)'
func getMajorMinorVer(osVersion string) (string, string) {
@@ -74,6 +83,12 @@ func (op Platform) ToGovalDictionaryFilename() string {
return fmt.Sprintf("%s_%s.sqlite3", GovalDictionaryFilePrefix, op)
}
// ToGovalDatabaseFilename returns the filename of the sqlite3 files downloaded using
// the goval-dictionary fetch method in the vulnerabilities generate-cve.yml workflow
func (op Platform) ToGovalDatabaseFilename() string {
return fmt.Sprintf("%s.sqlite3", op)
}
// IsSupported returns whether the given platform is currently supported.
func (op Platform) IsSupported() bool {
supported := []string{
@@ -104,14 +119,7 @@ func (op Platform) IsSupported() bool {
}
func (op Platform) IsGovalDictionarySupported() bool {
supported := []string{
"amzn_01",
"amzn_02",
"amzn_2022",
"amzn_2023",
}
for _, p := range supported {
for _, p := range SupportedGovalPlatforms {
if strings.HasPrefix(string(op), p) {
return true
}
server/vulnerabilities/oval/oval_platform_test.go
+16
View File
@@ -88,4 +88,20 @@ func TestOvalPlatform(t *testing.T) {
require.Equal(t, c.expected, plat.ToGovalDictionaryFilename())
}
})
t.Run("ToGovalDatabaseFilename", func(t *testing.T) {
cases := []struct {
version string
expected string
}{
{"Amazon Linux 1.0.0", "amzn_01.sqlite3"},
{"Amazon Linux 2.0.0", "amzn_02.sqlite3"},
{"Amazon Linux 2022.0.0", "amzn_2022.sqlite3"},
{"Amazon Linux 2023.0.0", "amzn_2023.sqlite3"},
}
for _, c := range cases {
plat := NewPlatform("amzn", c.version)
require.Equal(t, c.expected, plat.ToGovalDatabaseFilename())
}
})
}