Discharge7443/fleet
1
0
Fork 0
Code Issues Pull Requests Actions 22 Packages Projects Releases Wiki Activity

Ubuntu Python Package Filtering (#21989)

Browse Source
This commit is contained in:
Tim Lee
2024-09-16 10:01:21 -06:00
committed by GitHub
parent e41cfe9289
commit 1da93d4c3c
3 changed files with 257 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
changes/19442-ubuntu-python-packages
+1
View File
@@ -0,0 +1 @@
- Addressing Ubuntu python package false positive vulnerabilities by removing duplicate entries for ubuntu python packages installed by dpkg and renaming remaining pip installed packages to match OVAL definitions.
server/service/osquery.go
+85 -4
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"net/http"
"regexp"
"sort"
"strconv"
"strings"
"sync/atomic"
@@ -955,7 +956,7 @@ func (svc *Service) SubmitDistributedQueryResults(
svc.maybeDebugHost(ctx, host, results, statuses, messages, stats)
preProcessSoftwareResults(host.ID, &results, &statuses, &messages, osquery_utils.SoftwareOverrideQueries, svc.logger)
preProcessSoftwareResults(host, &results, &statuses, &messages, osquery_utils.SoftwareOverrideQueries, svc.logger)
var hostWithoutPolicies bool
for query, rows := range results {
@@ -1232,20 +1233,100 @@ func getFailingCalendarPolicies(policyResults map[uint]*bool, calendarPolicies [
// We do this to not grow the main software queries and to ingest
// all software together (one direct ingest function for all software).
func preProcessSoftwareResults(
hostID uint,
host *fleet.Host,
results *fleet.OsqueryDistributedQueryResults,
statuses *map[string]fleet.OsqueryStatus,
messages *map[string]string,
overrides map[string]osquery_utils.DetailQuery,
logger log.Logger,
) {
//
vsCodeExtensionsExtraQuery := hostDetailQueryPrefix + "software_vscode_extensions"
preProcessSoftwareExtraResults(vsCodeExtensionsExtraQuery, hostID, results, statuses, messages, osquery_utils.DetailQuery{}, logger)
preProcessSoftwareExtraResults(vsCodeExtensionsExtraQuery, host.ID, results, statuses, messages, osquery_utils.DetailQuery{}, logger)
for name, query := range overrides {
fullQueryName := hostDetailQueryPrefix + "software_" + name
preProcessSoftwareExtraResults(fullQueryName, hostID, results, statuses, messages, query, logger)
preProcessSoftwareExtraResults(fullQueryName, host.ID, results, statuses, messages, query, logger)
}
// Filter out python packages that are also deb packages on ubuntu
pythonPackageFilter(host.Platform, results, statuses)
}
// pythonPackageFilter filters out duplicate python_packages that are installed under deb_packages on Ubuntu.
// python_packages not matching a Debian package names are updated to "python3-packagename" to match OVAL definitions.
func pythonPackageFilter(platform string, results *fleet.OsqueryDistributedQueryResults, statuses *map[string]fleet.OsqueryStatus) {
const pythonPrefix = "python3-"
const pythonSource = "python_packages"
const debSource = "deb_packages"
const linuxSoftware = hostDetailQueryPrefix + "software_linux"
// Return early if platform is not Ubuntu
// We may need to add more platforms in the future
if platform != "ubuntu" {
return
}
// Check the 'software_linux' result and status
sw, ok := (*results)[linuxSoftware]
if !ok {
return
}
if status, ok := (*statuses)[linuxSoftware]; !ok || status != fleet.StatusOK {
return
}
// Extract the Python and Debian packages from the software list for filtering
// pre-allocating space for 40 packages based on number of package found in
// a fresh ubuntu 24.04 install
pythonPackages := make(map[string]int, 40)
debPackages := make(map[string]struct{}, 40)
// Track indexes of rows to remove
indexesToRemove := []int{}
for i, row := range sw {
switch row["source"] {
case pythonSource:
loweredName := strings.ToLower(row["name"])
pythonPackages[loweredName] = i
row["name"] = loweredName
case debSource:
// Only append python3 deb packages
if strings.HasPrefix(row["name"], pythonPrefix) {
debPackages[row["name"]] = struct{}{}
}
}
}
// Return early if there are no Python packages to process
if len(pythonPackages) == 0 {
return
}
// Loop through pythonPackages map to identify any that should be removed
for name, index := range pythonPackages {
convertedName := pythonPrefix + name
// Filter out Python packages that are also Debian packages
if _, found := debPackages[convertedName]; found {
indexesToRemove = append(indexesToRemove, index)
} else {
// Update remaining Python package names to match OVAL definitions
sw[index]["name"] = convertedName
}
}
// Sort indexes to remove in descending order
sort.Sort(sort.Reverse(sort.IntSlice(indexesToRemove)))
// Remove rows from sw in descending order of indexes
for _, index := range indexesToRemove {
sw = append(sw[:index], sw[index+1:]...)
}
// Store the updated software result back in the results map
(*results)[linuxSoftware] = sw
}
func preProcessSoftwareExtraResults(
server/service/osquery_test.go
+171 -3
View File
@@ -3678,8 +3678,8 @@ func TestPreProcessSoftwareResults(t *testing.T) {
}
for _, tc := range []struct {
name string
name string
host *fleet.Host
resultsIn fleet.OsqueryDistributedQueryResults
statusesIn map[string]fleet.OsqueryStatus
messagesIn map[string]string
@@ -3898,10 +3898,134 @@ func TestPreProcessSoftwareResults(t *testing.T) {
},
},
},
{
name: "ubuntu dpkg installed python packages are filtered out",
host: &fleet.Host{ID: 1, Platform: "ubuntu"},
statusesIn: map[string]fleet.OsqueryStatus{
hostDetailQueryPrefix + "software_linux": fleet.StatusOK,
},
resultsIn: fleet.OsqueryDistributedQueryResults{
hostDetailQueryPrefix + "software_linux": []map[string]string{
{
"name": "python3-twisted",
"version": "20.3.0-2",
"source": "deb_packages",
},
{
"name": "Twisted", // duplicate of python3-twisted
"version": "20.3.0-2",
"source": "python_packages",
},
{
"name": "python3-setuptools",
"version": "50.3.2",
"source": "deb_packages",
},
{
"name": "setuptools",
"version": "50.3.2",
"source": "python_packages",
},
{
"name": "pillow",
"version": "8.1.0",
"source": "python_packages",
},
{
"name": "python3-urllib3",
"version": "1.26.2-2",
"source": "deb_packages",
},
},
},
resultsOut: fleet.OsqueryDistributedQueryResults{
hostDetailQueryPrefix + "software_linux": []map[string]string{
{
"name": "python3-twisted",
"version": "20.3.0-2",
"source": "deb_packages",
},
{
"name": "python3-setuptools",
"version": "50.3.2",
"source": "deb_packages",
},
{
"name": "python3-pillow", // renamed from pillow
"version": "8.1.0",
"source": "python_packages",
},
{
"name": "python3-urllib3",
"version": "1.26.2-2",
"source": "deb_packages",
},
},
},
},
{
name: "non-ubuntu installed python packages are NOT filtered out",
host: &fleet.Host{ID: 1, Platform: "rhel"},
statusesIn: map[string]fleet.OsqueryStatus{
hostDetailQueryPrefix + "software_linux": fleet.StatusOK,
},
resultsIn: fleet.OsqueryDistributedQueryResults{
hostDetailQueryPrefix + "software_linux": []map[string]string{
{
"name": "python3-twisted",
"version": "20.3.0-2",
"source": "rpm_packages",
},
{
"name": "twisted", // duplicate of python3-twisted
"version": "20.3.0-2",
"source": "python_packages",
},
{
"name": "pillow",
"version": "8.1.0",
"source": "python_packages",
},
{
"name": "python3-urllib3",
"version": "1.26.2-2",
"source": "rpm_packages",
},
},
},
resultsOut: fleet.OsqueryDistributedQueryResults{
hostDetailQueryPrefix + "software_linux": []map[string]string{
{
"name": "python3-twisted",
"version": "20.3.0-2",
"source": "rpm_packages",
},
{
"name": "twisted", // duplicate of python3-twisted
"version": "20.3.0-2",
"source": "python_packages",
},
{
"name": "pillow",
"version": "8.1.0",
"source": "python_packages",
},
{
"name": "python3-urllib3",
"version": "1.26.2-2",
"source": "rpm_packages",
},
},
},
},
} {
tc := tc
t.Run(tc.name, func(t *testing.T) {
preProcessSoftwareResults(1, &tc.resultsIn, &tc.statusesIn, &tc.messagesIn, tc.overrides, log.NewNopLogger())
host := &fleet.Host{ID: 1}
if tc.host != nil {
host = tc.host
}
preProcessSoftwareResults(host, &tc.resultsIn, &tc.statusesIn, &tc.messagesIn, tc.overrides, log.NewNopLogger())
require.Equal(t, tc.resultsOut, tc.resultsIn)
})
}
@@ -3943,3 +4067,47 @@ func BenchmarkFindPackDelimiterStringTeamPack(b *testing.B) {
findPackDelimiterString(input)
}
}
func mockUbuntuResults() *fleet.OsqueryDistributedQueryResults {
results := &fleet.OsqueryDistributedQueryResults{
hostDetailQueryPrefix + "software_linux": make([]map[string]string, 0),
}
// Adding 40 python packages with matching deb packages
// Adding 2 python packages without matching deb packages
for i := 1; i <= 42; i++ {
pythonPkg := fmt.Sprintf("package%d", i)
(*results)[hostDetailQueryPrefix+"software_linux"] = append((*results)[hostDetailQueryPrefix+"software_linux"], map[string]string{
"source": "python_packages",
"name": pythonPkg,
})
}
// Adding 1500 deb packages, with the first 40 matching python packages
for i := 1; i <= 1500; i++ {
var debPkg string
if i <= 38 { // Match first 38 python packages
debPkg = fmt.Sprintf("python3-package%d", i)
} else { // Non-python packages
debPkg = fmt.Sprintf("unrelated_package%d", i)
}
(*results)[hostDetailQueryPrefix+"software_linux"] = append((*results)[hostDetailQueryPrefix+"software_linux"], map[string]string{
"source": "deb_packages",
"name": debPkg,
})
}
return results
}
func BenchmarkPreprocessUbuntuPythonPackageFilter(b *testing.B) {
platform := "ubuntu"
results := mockUbuntuResults()
statuses := &map[string]fleet.OsqueryStatus{
hostDetailQueryPrefix + "software_linux": fleet.StatusOK,
}
for i := 0; i < b.N; i++ {
preProcessSoftwareResults(&fleet.Host{ID: 1, Platform: platform}, results, statuses, nil, nil, log.NewNopLogger())
}
}