diff --git a/changes/19442-ubuntu-python-packages b/changes/19442-ubuntu-python-packages new file mode 100644 index 0000000000..0be7e95616 --- /dev/null +++ b/changes/19442-ubuntu-python-packages @@ -0,0 +1 @@ +- Addressing Ubuntu python package false positive vulnerabilities by removing duplicate entries for ubuntu python packages installed by dpkg and renaming remaining pip installed packages to match OVAL definitions. \ No newline at end of file diff --git a/server/service/osquery.go b/server/service/osquery.go index d3fb9c6920..de5c1e2e26 100644 --- a/server/service/osquery.go +++ b/server/service/osquery.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "regexp" + "sort" "strconv" "strings" "sync/atomic" @@ -955,7 +956,7 @@ func (svc *Service) SubmitDistributedQueryResults( svc.maybeDebugHost(ctx, host, results, statuses, messages, stats) - preProcessSoftwareResults(host.ID, &results, &statuses, &messages, osquery_utils.SoftwareOverrideQueries, svc.logger) + preProcessSoftwareResults(host, &results, &statuses, &messages, osquery_utils.SoftwareOverrideQueries, svc.logger) var hostWithoutPolicies bool for query, rows := range results { @@ -1232,20 +1233,100 @@ func getFailingCalendarPolicies(policyResults map[uint]*bool, calendarPolicies [ // We do this to not grow the main software queries and to ingest // all software together (one direct ingest function for all software). func preProcessSoftwareResults( - hostID uint, + host *fleet.Host, results *fleet.OsqueryDistributedQueryResults, statuses *map[string]fleet.OsqueryStatus, messages *map[string]string, overrides map[string]osquery_utils.DetailQuery, logger log.Logger, ) { + // vsCodeExtensionsExtraQuery := hostDetailQueryPrefix + "software_vscode_extensions" - preProcessSoftwareExtraResults(vsCodeExtensionsExtraQuery, hostID, results, statuses, messages, osquery_utils.DetailQuery{}, logger) + preProcessSoftwareExtraResults(vsCodeExtensionsExtraQuery, host.ID, results, statuses, messages, osquery_utils.DetailQuery{}, logger) for name, query := range overrides { fullQueryName := hostDetailQueryPrefix + "software_" + name - preProcessSoftwareExtraResults(fullQueryName, hostID, results, statuses, messages, query, logger) + preProcessSoftwareExtraResults(fullQueryName, host.ID, results, statuses, messages, query, logger) } + + // Filter out python packages that are also deb packages on ubuntu + pythonPackageFilter(host.Platform, results, statuses) +} + +// pythonPackageFilter filters out duplicate python_packages that are installed under deb_packages on Ubuntu. +// python_packages not matching a Debian package names are updated to "python3-packagename" to match OVAL definitions. +func pythonPackageFilter(platform string, results *fleet.OsqueryDistributedQueryResults, statuses *map[string]fleet.OsqueryStatus) { + const pythonPrefix = "python3-" + const pythonSource = "python_packages" + const debSource = "deb_packages" + const linuxSoftware = hostDetailQueryPrefix + "software_linux" + + // Return early if platform is not Ubuntu + // We may need to add more platforms in the future + if platform != "ubuntu" { + return + } + + // Check the 'software_linux' result and status + sw, ok := (*results)[linuxSoftware] + if !ok { + return + } + if status, ok := (*statuses)[linuxSoftware]; !ok || status != fleet.StatusOK { + return + } + + // Extract the Python and Debian packages from the software list for filtering + // pre-allocating space for 40 packages based on number of package found in + // a fresh ubuntu 24.04 install + pythonPackages := make(map[string]int, 40) + debPackages := make(map[string]struct{}, 40) + + // Track indexes of rows to remove + indexesToRemove := []int{} + + for i, row := range sw { + switch row["source"] { + case pythonSource: + loweredName := strings.ToLower(row["name"]) + pythonPackages[loweredName] = i + row["name"] = loweredName + case debSource: + // Only append python3 deb packages + if strings.HasPrefix(row["name"], pythonPrefix) { + debPackages[row["name"]] = struct{}{} + } + } + } + + // Return early if there are no Python packages to process + if len(pythonPackages) == 0 { + return + } + + // Loop through pythonPackages map to identify any that should be removed + for name, index := range pythonPackages { + convertedName := pythonPrefix + name + + // Filter out Python packages that are also Debian packages + if _, found := debPackages[convertedName]; found { + indexesToRemove = append(indexesToRemove, index) + } else { + // Update remaining Python package names to match OVAL definitions + sw[index]["name"] = convertedName + } + } + + // Sort indexes to remove in descending order + sort.Sort(sort.Reverse(sort.IntSlice(indexesToRemove))) + + // Remove rows from sw in descending order of indexes + for _, index := range indexesToRemove { + sw = append(sw[:index], sw[index+1:]...) + } + + // Store the updated software result back in the results map + (*results)[linuxSoftware] = sw } func preProcessSoftwareExtraResults( diff --git a/server/service/osquery_test.go b/server/service/osquery_test.go index 8ceb4ec77b..0697c7de69 100644 --- a/server/service/osquery_test.go +++ b/server/service/osquery_test.go @@ -3678,8 +3678,8 @@ func TestPreProcessSoftwareResults(t *testing.T) { } for _, tc := range []struct { - name string - + name string + host *fleet.Host resultsIn fleet.OsqueryDistributedQueryResults statusesIn map[string]fleet.OsqueryStatus messagesIn map[string]string @@ -3898,10 +3898,134 @@ func TestPreProcessSoftwareResults(t *testing.T) { }, }, }, + { + name: "ubuntu dpkg installed python packages are filtered out", + host: &fleet.Host{ID: 1, Platform: "ubuntu"}, + statusesIn: map[string]fleet.OsqueryStatus{ + hostDetailQueryPrefix + "software_linux": fleet.StatusOK, + }, + resultsIn: fleet.OsqueryDistributedQueryResults{ + hostDetailQueryPrefix + "software_linux": []map[string]string{ + { + "name": "python3-twisted", + "version": "20.3.0-2", + "source": "deb_packages", + }, + { + "name": "Twisted", // duplicate of python3-twisted + "version": "20.3.0-2", + "source": "python_packages", + }, + { + "name": "python3-setuptools", + "version": "50.3.2", + "source": "deb_packages", + }, + { + "name": "setuptools", + "version": "50.3.2", + "source": "python_packages", + }, + { + "name": "pillow", + "version": "8.1.0", + "source": "python_packages", + }, + { + "name": "python3-urllib3", + "version": "1.26.2-2", + "source": "deb_packages", + }, + }, + }, + resultsOut: fleet.OsqueryDistributedQueryResults{ + hostDetailQueryPrefix + "software_linux": []map[string]string{ + { + "name": "python3-twisted", + "version": "20.3.0-2", + "source": "deb_packages", + }, + { + "name": "python3-setuptools", + "version": "50.3.2", + "source": "deb_packages", + }, + { + "name": "python3-pillow", // renamed from pillow + "version": "8.1.0", + "source": "python_packages", + }, + { + "name": "python3-urllib3", + "version": "1.26.2-2", + "source": "deb_packages", + }, + }, + }, + }, + { + name: "non-ubuntu installed python packages are NOT filtered out", + host: &fleet.Host{ID: 1, Platform: "rhel"}, + statusesIn: map[string]fleet.OsqueryStatus{ + hostDetailQueryPrefix + "software_linux": fleet.StatusOK, + }, + resultsIn: fleet.OsqueryDistributedQueryResults{ + hostDetailQueryPrefix + "software_linux": []map[string]string{ + { + "name": "python3-twisted", + "version": "20.3.0-2", + "source": "rpm_packages", + }, + { + "name": "twisted", // duplicate of python3-twisted + "version": "20.3.0-2", + "source": "python_packages", + }, + { + "name": "pillow", + "version": "8.1.0", + "source": "python_packages", + }, + { + "name": "python3-urllib3", + "version": "1.26.2-2", + "source": "rpm_packages", + }, + }, + }, + resultsOut: fleet.OsqueryDistributedQueryResults{ + hostDetailQueryPrefix + "software_linux": []map[string]string{ + { + "name": "python3-twisted", + "version": "20.3.0-2", + "source": "rpm_packages", + }, + { + "name": "twisted", // duplicate of python3-twisted + "version": "20.3.0-2", + "source": "python_packages", + }, + { + "name": "pillow", + "version": "8.1.0", + "source": "python_packages", + }, + { + "name": "python3-urllib3", + "version": "1.26.2-2", + "source": "rpm_packages", + }, + }, + }, + }, } { tc := tc t.Run(tc.name, func(t *testing.T) { - preProcessSoftwareResults(1, &tc.resultsIn, &tc.statusesIn, &tc.messagesIn, tc.overrides, log.NewNopLogger()) + host := &fleet.Host{ID: 1} + if tc.host != nil { + host = tc.host + } + preProcessSoftwareResults(host, &tc.resultsIn, &tc.statusesIn, &tc.messagesIn, tc.overrides, log.NewNopLogger()) require.Equal(t, tc.resultsOut, tc.resultsIn) }) } @@ -3943,3 +4067,47 @@ func BenchmarkFindPackDelimiterStringTeamPack(b *testing.B) { findPackDelimiterString(input) } } + +func mockUbuntuResults() *fleet.OsqueryDistributedQueryResults { + results := &fleet.OsqueryDistributedQueryResults{ + hostDetailQueryPrefix + "software_linux": make([]map[string]string, 0), + } + + // Adding 40 python packages with matching deb packages + // Adding 2 python packages without matching deb packages + for i := 1; i <= 42; i++ { + pythonPkg := fmt.Sprintf("package%d", i) + (*results)[hostDetailQueryPrefix+"software_linux"] = append((*results)[hostDetailQueryPrefix+"software_linux"], map[string]string{ + "source": "python_packages", + "name": pythonPkg, + }) + } + + // Adding 1500 deb packages, with the first 40 matching python packages + for i := 1; i <= 1500; i++ { + var debPkg string + if i <= 38 { // Match first 38 python packages + debPkg = fmt.Sprintf("python3-package%d", i) + } else { // Non-python packages + debPkg = fmt.Sprintf("unrelated_package%d", i) + } + (*results)[hostDetailQueryPrefix+"software_linux"] = append((*results)[hostDetailQueryPrefix+"software_linux"], map[string]string{ + "source": "deb_packages", + "name": debPkg, + }) + } + + return results +} + +func BenchmarkPreprocessUbuntuPythonPackageFilter(b *testing.B) { + platform := "ubuntu" + results := mockUbuntuResults() + statuses := &map[string]fleet.OsqueryStatus{ + hostDetailQueryPrefix + "software_linux": fleet.StatusOK, + } + + for i := 0; i < b.N; i++ { + preProcessSoftwareResults(&fleet.Host{ID: 1, Platform: platform}, results, statuses, nil, nil, log.NewNopLogger()) + } +}