Custom Ubuntu Kernel Vuln Scanning (#19588)

This commit is contained in:
Tim Lee
2024-06-17 15:44:01 -06:00
committed by GitHub
parent c9f26b2e18
commit 2d6458b3f8
14 changed files with 330 additions and 48 deletions
@@ -0,0 +1 @@
- added vulnerability detection in NVD for custom ubuntu kernels
@@ -12,7 +12,6 @@ import (
)
func TestVulnerabilityDataStream(t *testing.T) {
t.Skip("REMOVEME: when API keys are restored")
nettest.Run(t)
runAppCheckErr(t, []string{"vulnerability-data-stream"}, "No directory provided")
+21 -15
View File
@@ -1140,33 +1140,39 @@ func (ds *Datastore) AllSoftwareIterator(
LEFT JOIN software_cpe sc ON (s.id=sc.software_id)`
var conditionals []string
arg := map[string]interface{}{}
if len(query.ExcludedSources) != 0 {
conditionals = append(conditionals, "s.source NOT IN (:excluded_sources)")
arg["excluded_sources"] = query.ExcludedSources
conditionals = append(conditionals, "s.source NOT IN (?)")
args = append(args, query.ExcludedSources)
}
if len(query.IncludedSources) != 0 {
conditionals = append(conditionals, "s.source IN (:included_sources)")
arg["included_sources"] = query.IncludedSources
conditionals = append(conditionals, "s.source IN (?)")
args = append(args, query.IncludedSources)
}
if query.NameMatch != "" {
conditionals = append(conditionals, "s.name REGEXP ?")
args = append(args, query.NameMatch)
}
if query.NameExclude != "" {
conditionals = append(conditionals, "s.name NOT REGEXP ?")
args = append(args, query.NameExclude)
}
if len(conditionals) != 0 {
cond := strings.Join(conditionals, " AND ")
stmt, args, err = sqlx.Named(stmt+" WHERE "+cond, arg)
if err != nil {
return nil, ctxerr.Wrap(ctx, err, "error binding named arguments on software iterator")
}
stmt, args, err = sqlx.In(stmt, args...)
if err != nil {
return nil, ctxerr.Wrap(ctx, err, "error building 'In' query part on software iterator")
}
stmt += " WHERE " + strings.Join(conditionals, " AND ")
}
stmt, args, err = sqlx.In(stmt, args...)
if err != nil {
return nil, fmt.Errorf("error building 'In' query part on software iterator: %w", err)
}
rows, err := ds.reader(ctx).QueryxContext(ctx, stmt, args...) //nolint:sqlclosecheck
if err != nil {
return nil, ctxerr.Wrap(ctx, err, "load host software")
return nil, fmt.Errorf("executing all software iterator %w", err)
}
return &softwareIterator{rows: rows}, nil
}
+91 -16
View File
@@ -14,6 +14,7 @@ import (
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/ptr"
"github.com/fleetdm/fleet/v4/server/test"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd"
"github.com/fleetdm/fleet/v4/server/vulnerabilities/oval"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
@@ -51,6 +52,7 @@ func TestSoftware(t *testing.T) {
{"ListCVEs", testListCVEs},
{"ListSoftwareForVulnDetection", testListSoftwareForVulnDetection},
{"AllSoftwareIterator", testAllSoftwareIterator},
{"AllSoftwareIteratorForCustomLinuxImages", testSoftwareIteratorForLinuxKernelCustomImages},
{"UpsertSoftwareCPEs", testUpsertSoftwareCPEs},
{"DeleteOutOfDateVulnerabilities", testDeleteOutOfDateVulnerabilities},
{"DeleteSoftwareCPEs", testDeleteSoftwareCPEs},
@@ -2294,9 +2296,12 @@ func testAllSoftwareIterator(t *testing.T, ds *Datastore) {
software := []fleet.Software{
{Name: "foo", Version: "0.0.1", Source: "chrome_extensions"},
{Name: "foo", Version: "0.0.3", Source: "chrome_extensions"},
{Name: "foobar", Version: "0.0.1", Source: "chrome_extensions"},
{Name: "bar", Version: "0.0.3", Source: "chrome_extensions"},
{Name: "foo", Version: "v0.0.2", Source: "apps"},
{Name: "foo", Version: "0.0.3", Source: "apps"},
{Name: "bar", Version: "0.0.3", Source: "deb_packages"},
{Name: "baz", Version: "0.0.3", Source: "deb_packages"},
}
_, err := ds.UpdateHostSoftware(context.Background(), host.ID, software)
require.NoError(t, err)
@@ -2321,10 +2326,12 @@ func testAllSoftwareIterator(t *testing.T, ds *Datastore) {
require.NoError(t, err)
testCases := []struct {
name string
q fleet.SoftwareIterQueryOptions
expected []fleet.Software
}{
{
name: "include apps source",
expected: []fleet.Software{
{Name: "foo", Version: "v0.0.2", Source: "apps", GenerateCPE: "cpe:foo_app_v2"},
{Name: "foo", Version: "0.0.3", Source: "apps"},
@@ -2332,47 +2339,115 @@ func testAllSoftwareIterator(t *testing.T, ds *Datastore) {
q: fleet.SoftwareIterQueryOptions{IncludedSources: []string{"apps"}},
},
{
name: "exclude apps source",
expected: []fleet.Software{
{Name: "foo", Version: "0.0.1", Source: "chrome_extensions", GenerateCPE: "cpe:foo_ce_v1"},
{Name: "foo", Version: "0.0.3", Source: "chrome_extensions"},
{Name: "bar", Version: "0.0.3", Source: "chrome_extensions"},
{Name: "foobar", Version: "0.0.1", Source: "chrome_extensions"},
{Name: "bar", Version: "0.0.3", Source: "deb_packages", GenerateCPE: "cpe:bar_v3"},
{Name: "baz", Version: "0.0.3", Source: "deb_packages"},
},
q: fleet.SoftwareIterQueryOptions{ExcludedSources: []string{"apps"}},
},
{
expected: []fleet.Software{
{Name: "foo", Version: "v0.0.2", Source: "apps", GenerateCPE: "cpe:foo_app_v2"},
{Name: "foo", Version: "0.0.3", Source: "apps"},
},
q: fleet.SoftwareIterQueryOptions{IncludedSources: []string{"apps"}},
},
{
name: "no filter",
expected: []fleet.Software{
{Name: "foo", Version: "0.0.1", Source: "chrome_extensions", GenerateCPE: "cpe:foo_ce_v1"},
{Name: "foo", Version: "v0.0.2", Source: "apps", GenerateCPE: "cpe:foo_app_v2"},
{Name: "foo", Version: "0.0.3", Source: "apps"},
{Name: "foo", Version: "0.0.3", Source: "chrome_extensions"},
{Name: "bar", Version: "0.0.3", Source: "chrome_extensions"},
{Name: "foobar", Version: "0.0.1", Source: "chrome_extensions"},
{Name: "baz", Version: "0.0.3", Source: "deb_packages"},
{Name: "bar", Version: "0.0.3", Source: "deb_packages", GenerateCPE: "cpe:bar_v3"},
},
q: fleet.SoftwareIterQueryOptions{},
},
{
name: "partial name filter includes deb_packages",
expected: []fleet.Software{
{Name: "bar", Version: "0.0.3", Source: "deb_packages", GenerateCPE: "cpe:bar_v3"},
},
q: fleet.SoftwareIterQueryOptions{NameMatch: `ba[r|f]`, IncludedSources: []string{"deb_packages"}},
},
{
name: "name filter includes chrome_extensions",
expected: []fleet.Software{
{Name: "foo", Version: "0.0.1", Source: "chrome_extensions", GenerateCPE: "cpe:foo_ce_v1"},
{Name: "foo", Version: "0.0.3", Source: "chrome_extensions"},
{Name: "foobar", Version: "0.0.1", Source: "chrome_extensions"},
},
q: fleet.SoftwareIterQueryOptions{NameMatch: "foo\\.*", IncludedSources: []string{"chrome_extensions"}},
},
{
name: "name filter and not name filter",
expected: []fleet.Software{
{Name: "foo", Version: "0.0.1", Source: "chrome_extensions", GenerateCPE: "cpe:foo_ce_v1"},
{Name: "foo", Version: "0.0.3", Source: "chrome_extensions"},
},
q: fleet.SoftwareIterQueryOptions{NameMatch: "foo\\.*", NameExclude: "bar$", IncludedSources: []string{"chrome_extensions"}},
},
}
for _, tC := range testCases {
var actual []fleet.Software
t.Run(tC.name, func(t *testing.T) {
var actual []fleet.Software
iter, err := ds.AllSoftwareIterator(context.Background(), tC.q)
require.NoError(t, err)
for iter.Next() {
software, err := iter.Value()
iter, err := ds.AllSoftwareIterator(context.Background(), tC.q)
require.NoError(t, err)
actual = append(actual, *software)
}
iter.Close()
test.ElementsMatchSkipID(t, tC.expected, actual)
for iter.Next() {
software, err := iter.Value()
require.NoError(t, err)
actual = append(actual, *software)
}
iter.Close()
test.ElementsMatchSkipID(t, tC.expected, actual)
})
}
}
func testSoftwareIteratorForLinuxKernelCustomImages(t *testing.T, ds *Datastore) {
host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now())
software := []fleet.Software{
{Name: "linux-image-5.4.0-42-generic", Version: "5.4.0-42.46", Source: "deb_packages"},
{Name: "linux-image-6.5.0-42-generic", Version: "6.5.0-100.27", Source: "deb_packages"},
{Name: "linux-image-5.4.0-42-custom", Version: "5.4.0-42.46", Source: "deb_packages"},
{Name: "linux-image-6.5.0-42-1234-foo", Version: "6.5.0-100.27", Source: "deb_packages"},
{Name: "linux-image-generic", Version: "1.0.0", Source: "deb_packages"},
{Name: "foo", Version: "0.0.1", Source: "chrome_extensions"},
{Name: "bar", Version: "0.0.3", Source: "deb_packages"},
}
_, err := ds.UpdateHostSoftware(context.Background(), host.ID, software)
require.NoError(t, err)
require.NoError(t, ds.LoadHostSoftware(context.Background(), host, false))
expected := []fleet.Software{
{Name: "linux-image-5.4.0-42-custom", Version: "5.4.0-42.46", Source: "deb_packages"},
{Name: "linux-image-6.5.0-42-1234-foo", Version: "6.5.0-100.27", Source: "deb_packages"},
}
opts := fleet.SoftwareIterQueryOptions{
NameMatch: nvd.LinuxImageRegex,
NameExclude: nvd.BuildLinuxExclusionRegex(),
IncludedSources: []string{"deb_packages"},
}
iterator, err := ds.AllSoftwareIterator(context.Background(), opts)
require.NoError(t, err)
var actual []fleet.Software
for iterator.Next() {
software, err := iterator.Value()
require.NoError(t, err)
actual = append(actual, *software)
}
iterator.Close()
test.ElementsMatchSkipID(t, expected, actual)
}
func testUpsertSoftwareCPEs(t *testing.T, ds *Datastore) {
ctx := context.Background()
host := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now())
+2
View File
@@ -267,6 +267,8 @@ type SoftwareListOptions struct {
type SoftwareIterQueryOptions struct {
ExcludedSources []string // what sources to exclude
IncludedSources []string // what sources to include
NameMatch string // mysql regex to filter software by name
NameExclude string // mysql regex to filter software by name
}
// IsValid checks that either ExcludedSources or IncludedSources is specified but not both
@@ -12,7 +12,6 @@ import (
)
func TestIntegrationSync(t *testing.T) {
t.Skip("REMOVEME: when API keys are restored")
nettest.Run(t)
vulnPath := t.TempDir()
+65 -5
View File
@@ -1,9 +1,66 @@
# Testing CPE Translations
# CPE Translations
To improve accuracy when [mapping software to CVEs](../../../docs/Using%20Fleet/Vulnerability-Processing.md), we can add data to [cpe_translations.json](./cpe_translations.json) which
will get picked up by the NVD repo.
CPE Translations are rules to address bugs when translating Fleet software to Common Platform Enumerations
(CPEs) which are used to identify software in the National Vulnerability Database (NVD)
To test these changes locally, you can:
To improve accuracy when [mapping software to CVEs](../../../docs/Using%20Fleet/Vulnerability-Processing.md), we can add data to [cpe_translations.json](./cpe_translations.json)
## How CPE translations work
CPE Translations are defined in `cpe_translations.json` and currently released in
[GitHub](https://github.com/fleetdm/nvd) once a day. The rules are specified in JSON format and
and each rule consists of a `software` and a `filter` object.
`software` defines matching logic on what Fleet Software this rule should apply to. You can use one
or more of the below attributes to match on. Each attribute is an array of string or regex
matches (a regex string is identified by a leading and trailing `/`).
A match on the attribute is found if at least 1 item in the array matches. If multiple
attributes are defined, then a match is needed for each attribute. (ie. `name == Zoom.app` &&
`source == apps`)
`software` attributes:
- `name`: A software name attribute
- `bundle_identifier`: A software bundle_identifier attribute (macOS only)
- `source`: A software source attribute (ie. `apps`, `chrome_extensions`, etc...)
**example**: Search Fleet software for items that match: (bundle_identifier == us.zoom.xos) AND (source = apps)
```json
"software": {
"bundle_identifier": ["us.zoom.xos"],
"source": ["apps"]
}
```
If the software rule matches, then Fleet will search known NVD CPEs (stored in a local sqlite database) using the
specified filters or skip the software item based on the `filter` specified.
`filter` attributes:
- `product`: array of strings to search by product field
- `vendor`: array of strings to search by vendor field
- `target_sw`: array of strings to search by target_sw field
- `part`: string to override the default "a" Part value
- `skip`: boolean; software is skipped if `true`. This overrides any other filters set.
Like the software matching logic, filter items are matched by OR within the array, and AND between
filter items
**example**: Query the CPE database for a CPE that matches:
(product == zoom OR product == meetings) AND (vendor == zoom) AND (target == macos OR target == mac_os)
```json
"filter": {
"product": ["zoom", "meetings"],
"vendor": ["zoom"],
"target_sw": ["macos", "mac_os"]
}
```
## Testing CPE Translations (end-to-end)
1. make the [appropriate](../../../docs/Using%20Fleet/Vulnerability-Processing.md#Improving-accuracy) changes to cpe_translations
@@ -16,17 +73,20 @@ To test these changes locally, you can:
3. (re)launch your local fleet server with one of the following
Config method
```yaml
vulnerabilities:
cpe_translations_url: "http://localhost:8082/cpe_translations.json"
```
Environment method
```bash
FLEET_VULNERABILITIES_CPE_TRANSLATIONS_URL="http://localhost:8082/cpe_translations.json" ./build/fleet serve --dev --dev_license --logging_debug
```
4. trigger a vulnerabilities scan
```bash
fleetctl trigger --name vulnerabilities
```
+102 -5
View File
@@ -241,6 +241,9 @@ func CPEFromSoftware(logger log.Logger, db *sqlx.DB, software *fleet.Software, t
}
if result.ID != 0 {
if translation.Part != "" {
result.Part = translation.Part
}
return result.FmtStr(software), nil
}
} else {
@@ -377,25 +380,119 @@ func consumeCPEBuffer(
return nil
}
// mysql 5.7 compatible regexp for ubuntu kernel package names
const LinuxImageRegex = `^linux-image-[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+-[[:digit:]]+-[[:alnum:]]+`
// knownUbuntuKernelVariants is a list of known kernel variants that are used in the Ubuntu kernel
// OVAL feeds. These are used to determine if a kernel package is a custom variant and should be
// matched against the NVD feed rather than the OVAL feed.
var knownUbuntuKernelVariants = []string{
"allwinner",
"aws",
"aws-hwe",
"azure",
"azure-fde",
"bluefield",
"dell300x",
"euclid",
"gcp",
"generic",
"generic-64k",
"generic-lpae",
"gke",
"gkeop",
"intel",
"intel-iotg",
"ibm",
"iot",
"kvm",
"laptop",
"lowlatency",
"lowlatency-64k",
"nvidia",
"nvidia-64k",
"nvidia-lowlatency",
"oem",
"oem-osp1",
"oracle",
"oracle-64k",
"powerpc-e500",
"powerpc-e500mc",
"powerpc-smp",
"powerpc64-emb",
"powerpc64-smp",
"raspi",
"raspi-nolpae",
"raspi2",
"snapdragon",
"starfive",
"xilinx-zynqmp",
}
func BuildLinuxExclusionRegex() string {
return fmt.Sprintf("-(%s)$", strings.Join(knownUbuntuKernelVariants, "|"))
}
func TranslateSoftwareToCPE(
ctx context.Context,
ds fleet.Datastore,
vulnPath string,
logger kitlog.Logger,
) error {
dbPath := filepath.Join(vulnPath, cpeDBFilename)
// Skip software from sources for which we will be using OVAL for vulnerability detection.
iterator, err := ds.AllSoftwareIterator(
nonOvalIterator, err := ds.AllSoftwareIterator(
ctx,
fleet.SoftwareIterQueryOptions{
ExcludedSources: oval.SupportedSoftwareSources,
},
)
if err != nil {
return ctxerr.Wrap(ctx, err, "software iterator")
return ctxerr.Wrap(ctx, err, "non-oval software iterator")
}
defer iterator.Close()
defer nonOvalIterator.Close()
err = translateSoftwareToCPEWithIterator(ctx, ds, vulnPath, logger, nonOvalIterator)
if err != nil {
return ctxerr.Wrap(ctx, err, "translate non-oval software to CPE")
}
if err := nonOvalIterator.Close(); err != nil {
return ctxerr.Wrap(ctx, err, "closing non-oval software iterator")
}
ubuntuKernelIterator, err := ds.AllSoftwareIterator(
ctx,
fleet.SoftwareIterQueryOptions{
IncludedSources: []string{"deb_packages"},
NameMatch: LinuxImageRegex,
NameExclude: BuildLinuxExclusionRegex(),
},
)
if err != nil {
return ctxerr.Wrap(ctx, err, "ubuntu kernel iterator")
}
defer ubuntuKernelIterator.Close()
err = translateSoftwareToCPEWithIterator(ctx, ds, vulnPath, logger, ubuntuKernelIterator)
if err != nil {
return ctxerr.Wrap(ctx, err, "translate ubuntu kernel to CPE")
}
if err := ubuntuKernelIterator.Close(); err != nil {
return ctxerr.Wrap(ctx, err, "closing ubuntu kernel iterator")
}
return nil
}
func translateSoftwareToCPEWithIterator(
ctx context.Context,
ds fleet.Datastore,
vulnPath string,
logger kitlog.Logger,
iterator fleet.SoftwareIterator,
) error {
dbPath := filepath.Join(vulnPath, cpeDBFilename)
db, err := sqliteDB(dbPath)
if err != nil {
+31 -2
View File
@@ -135,6 +135,28 @@ func TestCPETranslations(t *testing.T) {
},
Expected: "cpe:2.3:a:vendor:product-1:1.2.3:*:*:*:*:macos:*:*",
},
{
Name: "translate part",
Translations: CPETranslations{
{
Software: CPETranslationSoftware{
Name: []string{"X"},
Source: []string{"apps"},
},
Filter: CPETranslation{
Product: []string{"product-1"},
Vendor: []string{"vendor"},
Part: "o",
},
},
},
Software: &fleet.Software{
Name: "X",
Version: "1.2.3",
Source: "apps",
},
Expected: "cpe:2.3:o:vendor:product-1:1.2.3:*:*:*:*:macos:*:*",
},
}
reCache := newRegexpCache()
@@ -149,7 +171,6 @@ func TestCPETranslations(t *testing.T) {
}
func TestSyncCPEDatabase(t *testing.T) {
t.Skip("REMOVEME: when API keys are restored")
nettest.Run(t)
tempDir := t.TempDir()
@@ -467,7 +488,6 @@ func TestLegacyCPEDB(t *testing.T) {
}
func TestCPEFromSoftwareIntegration(t *testing.T) {
t.Skip("REMOVEME: when API keys are restored")
testCases := []struct {
software fleet.Software
cpe string
@@ -1615,6 +1635,15 @@ func TestCPEFromSoftwareIntegration(t *testing.T) {
},
cpe: `cpe:2.3:a:python:python:3.9.18_2:*:*:*:*:*:*:*`,
},
{
software: fleet.Software{
Name: "linux-image-5.4.0-105-custom",
Source: "deb_packages",
Version: "5.4.0-105.118",
Vendor: "",
},
cpe: "cpe:2.3:o:linux:linux_kernel:5.4.0-105.118:*:*:*:*:*:*:*",
},
}
// NVD_TEST_CPEDB_PATH can be used to speed up development (sync cpe.sqlite only once).
@@ -215,6 +215,7 @@ type CPETranslation struct {
Product []string `json:"product"`
Vendor []string `json:"vendor"`
TargetSW []string `json:"target_sw"`
Part string `json:"part"`
// If Skip is set, no NVD vulnerabilities will be reported for the matching software.
Skip bool `json:"skip"`
}
@@ -377,5 +377,15 @@
"filter": {
"skip": true
}
},
{
"software": {
"name": ["/^linux-image\\.*/"]
},
"filter": {
"product": ["linux_kernel"],
"vendor": ["linux"],
"part": "o"
}
}
]
-1
View File
@@ -131,7 +131,6 @@ func (d *threadSafeDSMock) InsertSoftwareVulnerability(ctx context.Context, vuln
}
func TestTranslateCPEToCVE(t *testing.T) {
t.Skip("REMOVEME: when API keys are restored")
t.Parallel()
ctx := context.Background()
@@ -6,7 +6,8 @@ import (
)
type IndexedCPEItem struct {
ID int `json:"id" db:"rowid"`
ID int `json:"id" db:"rowid"`
Part string
Product string `json:"product" db:"product"`
Vendor string `json:"vendor" db:"vendor"`
Deprecated bool `json:"deprecated" db:"deprecated"`
@@ -21,6 +22,10 @@ func (i *IndexedCPEItem) FmtStr(s *fleet.Software) string {
cpe.Version = sanitizeVersion(s.Version)
cpe.TargetSW = targetSW(s)
if i.Part != "" {
cpe.Part = i.Part
}
// Make sure we don't return a 'match all' CPE
if cpe.Vendor == wfn.Any || cpe.Product == wfn.Any {
return ""
-1
View File
@@ -82,7 +82,6 @@ func TestLoadCVEMeta(t *testing.T) {
}
func TestDownloadCPETranslations(t *testing.T) {
t.Skip("REMOVEME: when API keys are restored")
nettest.Run(t)
tempDir := t.TempDir()