Discharge7443/fleet
1
0
Fork 0
Code Issues Pull Requests Actions 38 Packages Projects Releases Wiki Activity

Updated python_packages osquery query (#26434)

Browse Source 
A new feature in osquery `5.16` was created to allow for scanning of
user directories for python packages. If the new version of osquery is
detected use the new query, otherwise use the old query.

https://github.com/fleetdm/fleet/issues/26423

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/Committing-Changes.md#changes-files)
for more information.
- [x] Added/updated automated tests
- [x] A detailed QA plan exists on the associated ticket (if it isn't
there, work with the product group's QA engineer to add it)
- [x] Manual QA for all new/changed functionality
This commit is contained in:
Konstantin Sykulev
2025-02-26 14:15:41 -06:00
committed by GitHub
parent d690997f26
commit 124fc44a3e
5 changed files with 175 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/Contributing/Understanding-host-vitals.md
+53 -35
View File
@@ -567,19 +567,7 @@ SELECT
'' AS vendor,
'' AS arch,
path AS installed_path
FROM cached_users CROSS JOIN firefox_addons USING (uid)
UNION
SELECT
name AS name,
version AS version,
'' AS extension_id,
'' AS browser,
'python_packages' AS source,
'' AS release,
'' AS vendor,
'' AS arch,
path AS installed_path
FROM python_packages;
FROM cached_users CROSS JOIN firefox_addons USING (uid);
```
## software_macos
@@ -604,18 +592,6 @@ SELECT
path AS installed_path
FROM apps
UNION
SELECT
name AS name,
version AS version,
'' AS bundle_identifier,
'' AS extension_id,
'' AS browser,
'python_packages' AS source,
'' AS vendor,
0 AS last_opened_at,
path AS installed_path
FROM python_packages
UNION
SELECT
name AS name,
version AS version,
@@ -740,6 +716,58 @@ WITH app_paths AS (
WHERE apps.bundle_identifier = 'org.mozilla.firefox'
```
## software_python_packages
- Description: Prior to osquery version 5.16.0, the python_packages table did not search user directories.
- Platforms: linux, ubuntu, debian, rhel, centos, sles, kali, gentoo, amzn, pop, arch, linuxmint, void, nixos, endeavouros, manjaro, opensuse-leap, opensuse-tumbleweed, tuxedo, darwin, windows
- Discovery query:
```sql
SELECT 1 FROM osquery_info WHERE version_compare(version, '5.16.0') < 0
```
- Query:
```sql
SELECT
name AS name,
version AS version,
'' AS extension_id,
'' AS browser,
'python_packages' AS source,
'' AS vendor,
path AS installed_path
FROM python_packages
```
## software_python_packages_with_users_dir
- Description: As of osquery version 5.16.0, the python_packages table searches user directories with support from a cross join on users. See https://fleetdm.com/guides/osquery-consider-joining-against-the-users-table.
- Platforms: linux, ubuntu, debian, rhel, centos, sles, kali, gentoo, amzn, pop, arch, linuxmint, void, nixos, endeavouros, manjaro, opensuse-leap, opensuse-tumbleweed, tuxedo, darwin, windows
- Discovery query:
```sql
SELECT 1 FROM osquery_info WHERE version_compare(version, '5.16.0') >= 0
```
- Query:
```sql
WITH cached_users AS (WITH cached_groups AS (select * from groups)
SELECT uid, username, type, groupname, shell
FROM users LEFT JOIN cached_groups USING (gid)
WHERE type <> 'special' AND shell NOT LIKE '%/false' AND shell NOT LIKE '%/nologin' AND shell NOT LIKE '%/shutdown' AND shell NOT LIKE '%/halt' AND username NOT LIKE '%$' AND username NOT LIKE '\_%' ESCAPE '\' AND NOT (username = 'sync' AND shell ='/bin/sync' AND directory <> ''))
SELECT
name AS name,
version AS version,
'' AS extension_id,
'' AS browser,
'python_packages' AS source,
'' AS vendor,
path AS installed_path
FROM cached_users CROSS JOIN python_packages USING (uid)
```
## software_vscode_extensions
- Platforms: linux, ubuntu, debian, rhel, centos, sles, kali, gentoo, amzn, pop, arch, linuxmint, void, nixos, endeavouros, manjaro, opensuse-leap, opensuse-tumbleweed, tuxedo, darwin, windows
@@ -788,16 +816,6 @@ SELECT
install_location AS installed_path
FROM programs
UNION
SELECT
name AS name,
version AS version,
'' AS extension_id,
'' AS browser,
'python_packages' AS source,
'' AS vendor,
path AS installed_path
FROM python_packages
UNION
SELECT
name AS name,
version AS version,
server/service/osquery.go
+5
View File
@@ -1224,6 +1224,11 @@ func preProcessSoftwareResults(
vsCodeExtensionsExtraQuery := hostDetailQueryPrefix + "software_vscode_extensions"
preProcessSoftwareExtraResults(vsCodeExtensionsExtraQuery, host.ID, results, statuses, messages, osquery_utils.DetailQuery{}, logger)
pythonPackagesExtraQuery := hostDetailQueryPrefix + "software_python_packages"
preProcessSoftwareExtraResults(pythonPackagesExtraQuery, host.ID, results, statuses, messages, osquery_utils.DetailQuery{}, logger)
pythonPakcagesWithUsersExtraQuery := hostDetailQueryPrefix + "software_python_packages_with_users_dir"
preProcessSoftwareExtraResults(pythonPakcagesWithUsersExtraQuery, host.ID, results, statuses, messages, osquery_utils.DetailQuery{}, logger)
for name, query := range overrides {
fullQueryName := hostDetailQueryPrefix + "software_" + name
preProcessSoftwareExtraResults(fullQueryName, host.ID, results, statuses, messages, query, logger)
server/service/osquery_test.go
+75 -10
View File
@@ -1080,16 +1080,18 @@ func verifyDiscovery(t *testing.T, queries, discovery map[string]string) {
assert.Equal(t, len(queries), len(discovery))
// discoveryUsed holds the queries where we know use the distributed discovery feature.
discoveryUsed := map[string]struct{}{
hostDetailQueryPrefix + "google_chrome_profiles": {},
hostDetailQueryPrefix + "mdm": {},
hostDetailQueryPrefix + "munki_info": {},
hostDetailQueryPrefix + "windows_update_history": {},
hostDetailQueryPrefix + "kubequery_info": {},
hostDetailQueryPrefix + "orbit_info": {},
hostDetailQueryPrefix + "software_vscode_extensions": {},
hostDetailQueryPrefix + "software_macos_firefox": {},
hostDetailQueryPrefix + "battery": {},
hostDetailQueryPrefix + "software_macos_codesign": {},
hostDetailQueryPrefix + "google_chrome_profiles": {},
hostDetailQueryPrefix + "mdm": {},
hostDetailQueryPrefix + "munki_info": {},
hostDetailQueryPrefix + "windows_update_history": {},
hostDetailQueryPrefix + "kubequery_info": {},
hostDetailQueryPrefix + "orbit_info": {},
hostDetailQueryPrefix + "software_vscode_extensions": {},
hostDetailQueryPrefix + "software_python_packages": {},
hostDetailQueryPrefix + "software_python_packages_with_users_dir": {},
hostDetailQueryPrefix + "software_macos_firefox": {},
hostDetailQueryPrefix + "battery": {},
hostDetailQueryPrefix + "software_macos_codesign": {},
}
for name := range queries {
require.NotEmpty(t, discovery[name])
@@ -3709,6 +3711,25 @@ func TestPreProcessSoftwareResults(t *testing.T) {
"installed_path": "/some/override/path",
}
pythonPackageOne := map[string]string{
"name": "cryptography",
"version": "41.0.7",
"extension_id": "",
"browser": "",
"source": "python_packages",
"vendor": "",
"installed_path": "/usr/lib/python3/dist-packages",
}
pythonPackageTwo := map[string]string{
"name": "pip",
"version": "25.0.1",
"extension_id": "",
"browser": "",
"source": "python_packages",
"vendor": "",
"installed_path": "/Users/fleetdm/.pyenv/versions/3.13.1/lib/python3.13/site-packages",
}
for _, tc := range []struct {
name string
host *fleet.Host
@@ -3719,6 +3740,50 @@ func TestPreProcessSoftwareResults(t *testing.T) {
resultsOut fleet.OsqueryDistributedQueryResults
}{
{
name: "python packages using original query in extras adds results",
statusesIn: map[string]fleet.OsqueryStatus{
hostDetailQueryPrefix + "software_macos": fleet.StatusOK,
hostDetailQueryPrefix + "software_python_packages": fleet.StatusOK,
},
resultsIn: fleet.OsqueryDistributedQueryResults{
hostDetailQueryPrefix + "software_macos": []map[string]string{
foobarApp,
},
hostDetailQueryPrefix + "software_python_packages": []map[string]string{
pythonPackageOne,
},
},
resultsOut: fleet.OsqueryDistributedQueryResults{
hostDetailQueryPrefix + "software_macos": []map[string]string{
foobarApp,
pythonPackageOne,
},
},
},
{
name: "python packages using user query in extras adds results",
statusesIn: map[string]fleet.OsqueryStatus{
hostDetailQueryPrefix + "software_macos": fleet.StatusOK,
hostDetailQueryPrefix + "software_python_packages_with_users_dir": fleet.StatusOK,
},
resultsIn: fleet.OsqueryDistributedQueryResults{
hostDetailQueryPrefix + "software_macos": []map[string]string{
foobarApp,
},
hostDetailQueryPrefix + "software_python_packages_with_users_dir": []map[string]string{
pythonPackageTwo,
},
},
resultsOut: fleet.OsqueryDistributedQueryResults{
hostDetailQueryPrefix + "software_macos": []map[string]string{
foobarApp,
pythonPackageTwo,
},
},
},
{
name: "software query works and there are vs code extensions in extra",
server/service/osquery_utils/queries.go
+41 -35
View File
@@ -840,18 +840,6 @@ SELECT
path AS installed_path
FROM apps
UNION
SELECT
name AS name,
version AS version,
'' AS bundle_identifier,
'' AS extension_id,
'' AS browser,
'python_packages' AS source,
'' AS vendor,
0 AS last_opened_at,
path AS installed_path
FROM python_packages
UNION
SELECT
name AS name,
version AS version,
@@ -1024,19 +1012,7 @@ SELECT
'' AS vendor,
'' AS arch,
path AS installed_path
FROM cached_users CROSS JOIN firefox_addons USING (uid)
UNION
SELECT
name AS name,
version AS version,
'' AS extension_id,
'' AS browser,
'python_packages' AS source,
'' AS release,
'' AS vendor,
'' AS arch,
path AS installed_path
FROM python_packages;
FROM cached_users CROSS JOIN firefox_addons USING (uid);
`),
Platforms: fleet.HostLinuxOSs,
DirectIngestFunc: directIngestSoftware,
@@ -1054,16 +1030,6 @@ SELECT
install_location AS installed_path
FROM programs
UNION
SELECT
name AS name,
version AS version,
'' AS extension_id,
'' AS browser,
'python_packages' AS source,
'' AS vendor,
path AS installed_path
FROM python_packages
UNION
SELECT
name AS name,
version AS version,
@@ -1108,6 +1074,44 @@ FROM chocolatey_packages
DirectIngestFunc: directIngestSoftware,
}
// In osquery versions < 5.16.0 use the original python_packages query, as the cross join on
// users is not supported
var softwarePythonPackages = DetailQuery{
Description: "Prior to osquery version 5.16.0, the python_packages table did not search user directories.",
Query: `
SELECT
name AS name,
version AS version,
'' AS extension_id,
'' AS browser,
'python_packages' AS source,
'' AS vendor,
path AS installed_path
FROM python_packages
`,
Platforms: append(fleet.HostLinuxOSs, "darwin", "windows"),
Discovery: `SELECT 1 FROM osquery_info WHERE version_compare(version, '5.16.0') < 0`,
}
// In osquery versions >= 5.16.0 the python_packages table was modified to allow for a
// cross join on users so that user directories could be searched for python packages
var softwarePythonPackagesWithUsersDir = DetailQuery{
Description: "As of osquery version 5.16.0, the python_packages table searches user directories with support from a cross join on users. See https://fleetdm.com/guides/osquery-consider-joining-against-the-users-table.",
Query: withCachedUsers(`WITH cached_users AS (%s)
SELECT
name AS name,
version AS version,
'' AS extension_id,
'' AS browser,
'python_packages' AS source,
'' AS vendor,
path AS installed_path
FROM cached_users CROSS JOIN python_packages USING (uid)
`),
Platforms: append(fleet.HostLinuxOSs, "darwin", "windows"),
Discovery: `SELECT 1 FROM osquery_info WHERE version_compare(version, '5.16.0') >= 0`,
}
var softwareChrome = DetailQuery{
Query: `SELECT
name AS name,
@@ -2244,6 +2248,8 @@ func GetDetailQueries(
generatedMap["software_linux"] = softwareLinux
generatedMap["software_windows"] = softwareWindows
generatedMap["software_chrome"] = softwareChrome
generatedMap["software_python_packages"] = softwarePythonPackages
generatedMap["software_python_packages_with_users_dir"] = softwarePythonPackagesWithUsersDir
generatedMap["software_vscode_extensions"] = softwareVSCodeExtensions
for key, query := range SoftwareOverrideQueries {
server/service/osquery_utils/queries_test.go
+1 -1
View File
@@ -307,7 +307,7 @@ func TestGetDetailQueries(t *testing.T) {
queriesWithUsersAndSoftware := GetDetailQueries(context.Background(), config.FleetConfig{App: config.AppConfig{EnableScheduledQueryStats: true}}, nil, &fleet.Features{EnableHostUsers: true, EnableSoftwareInventory: true})
qs = baseQueries
qs = append(qs, "users", "users_chrome", "software_macos", "software_linux", "software_windows", "software_vscode_extensions",
"software_chrome", "scheduled_query_stats", "software_macos_firefox", "software_macos_codesign")
"software_chrome", "software_python_packages", "software_python_packages_with_users_dir", "scheduled_query_stats", "software_macos_firefox", "software_macos_codesign")
require.Len(t, queriesWithUsersAndSoftware, len(qs))
sortedKeysCompare(t, queriesWithUsersAndSoftware, qs)