diff --git a/docs/Contributing/Understanding-host-vitals.md b/docs/Contributing/Understanding-host-vitals.md index f4b6a1d362..dac415ef07 100644 --- a/docs/Contributing/Understanding-host-vitals.md +++ b/docs/Contributing/Understanding-host-vitals.md @@ -567,19 +567,7 @@ SELECT '' AS vendor, '' AS arch, path AS installed_path -FROM cached_users CROSS JOIN firefox_addons USING (uid) -UNION -SELECT - name AS name, - version AS version, - '' AS extension_id, - '' AS browser, - 'python_packages' AS source, - '' AS release, - '' AS vendor, - '' AS arch, - path AS installed_path -FROM python_packages; +FROM cached_users CROSS JOIN firefox_addons USING (uid); ``` ## software_macos @@ -604,18 +592,6 @@ SELECT path AS installed_path FROM apps UNION -SELECT - name AS name, - version AS version, - '' AS bundle_identifier, - '' AS extension_id, - '' AS browser, - 'python_packages' AS source, - '' AS vendor, - 0 AS last_opened_at, - path AS installed_path -FROM python_packages -UNION SELECT name AS name, version AS version, @@ -740,6 +716,58 @@ WITH app_paths AS ( WHERE apps.bundle_identifier = 'org.mozilla.firefox' ``` +## software_python_packages + +- Description: Prior to osquery version 5.16.0, the python_packages table did not search user directories. + +- Platforms: linux, ubuntu, debian, rhel, centos, sles, kali, gentoo, amzn, pop, arch, linuxmint, void, nixos, endeavouros, manjaro, opensuse-leap, opensuse-tumbleweed, tuxedo, darwin, windows + +- Discovery query: +```sql +SELECT 1 FROM osquery_info WHERE version_compare(version, '5.16.0') < 0 +``` + +- Query: +```sql +SELECT + name AS name, + version AS version, + '' AS extension_id, + '' AS browser, + 'python_packages' AS source, + '' AS vendor, + path AS installed_path + FROM python_packages +``` + +## software_python_packages_with_users_dir + +- Description: As of osquery version 5.16.0, the python_packages table searches user directories with support from a cross join on users. See https://fleetdm.com/guides/osquery-consider-joining-against-the-users-table. + +- Platforms: linux, ubuntu, debian, rhel, centos, sles, kali, gentoo, amzn, pop, arch, linuxmint, void, nixos, endeavouros, manjaro, opensuse-leap, opensuse-tumbleweed, tuxedo, darwin, windows + +- Discovery query: +```sql +SELECT 1 FROM osquery_info WHERE version_compare(version, '5.16.0') >= 0 +``` + +- Query: +```sql +WITH cached_users AS (WITH cached_groups AS (select * from groups) + SELECT uid, username, type, groupname, shell + FROM users LEFT JOIN cached_groups USING (gid) + WHERE type <> 'special' AND shell NOT LIKE '%/false' AND shell NOT LIKE '%/nologin' AND shell NOT LIKE '%/shutdown' AND shell NOT LIKE '%/halt' AND username NOT LIKE '%$' AND username NOT LIKE '\_%' ESCAPE '\' AND NOT (username = 'sync' AND shell ='/bin/sync' AND directory <> '')) + SELECT + name AS name, + version AS version, + '' AS extension_id, + '' AS browser, + 'python_packages' AS source, + '' AS vendor, + path AS installed_path + FROM cached_users CROSS JOIN python_packages USING (uid) +``` + ## software_vscode_extensions - Platforms: linux, ubuntu, debian, rhel, centos, sles, kali, gentoo, amzn, pop, arch, linuxmint, void, nixos, endeavouros, manjaro, opensuse-leap, opensuse-tumbleweed, tuxedo, darwin, windows @@ -788,16 +816,6 @@ SELECT install_location AS installed_path FROM programs UNION -SELECT - name AS name, - version AS version, - '' AS extension_id, - '' AS browser, - 'python_packages' AS source, - '' AS vendor, - path AS installed_path -FROM python_packages -UNION SELECT name AS name, version AS version, diff --git a/server/service/osquery.go b/server/service/osquery.go index ec9848d467..5df874a6e0 100644 --- a/server/service/osquery.go +++ b/server/service/osquery.go @@ -1224,6 +1224,11 @@ func preProcessSoftwareResults( vsCodeExtensionsExtraQuery := hostDetailQueryPrefix + "software_vscode_extensions" preProcessSoftwareExtraResults(vsCodeExtensionsExtraQuery, host.ID, results, statuses, messages, osquery_utils.DetailQuery{}, logger) + pythonPackagesExtraQuery := hostDetailQueryPrefix + "software_python_packages" + preProcessSoftwareExtraResults(pythonPackagesExtraQuery, host.ID, results, statuses, messages, osquery_utils.DetailQuery{}, logger) + pythonPakcagesWithUsersExtraQuery := hostDetailQueryPrefix + "software_python_packages_with_users_dir" + preProcessSoftwareExtraResults(pythonPakcagesWithUsersExtraQuery, host.ID, results, statuses, messages, osquery_utils.DetailQuery{}, logger) + for name, query := range overrides { fullQueryName := hostDetailQueryPrefix + "software_" + name preProcessSoftwareExtraResults(fullQueryName, host.ID, results, statuses, messages, query, logger) diff --git a/server/service/osquery_test.go b/server/service/osquery_test.go index b8dfbbc636..15f389ddb2 100644 --- a/server/service/osquery_test.go +++ b/server/service/osquery_test.go @@ -1080,16 +1080,18 @@ func verifyDiscovery(t *testing.T, queries, discovery map[string]string) { assert.Equal(t, len(queries), len(discovery)) // discoveryUsed holds the queries where we know use the distributed discovery feature. discoveryUsed := map[string]struct{}{ - hostDetailQueryPrefix + "google_chrome_profiles": {}, - hostDetailQueryPrefix + "mdm": {}, - hostDetailQueryPrefix + "munki_info": {}, - hostDetailQueryPrefix + "windows_update_history": {}, - hostDetailQueryPrefix + "kubequery_info": {}, - hostDetailQueryPrefix + "orbit_info": {}, - hostDetailQueryPrefix + "software_vscode_extensions": {}, - hostDetailQueryPrefix + "software_macos_firefox": {}, - hostDetailQueryPrefix + "battery": {}, - hostDetailQueryPrefix + "software_macos_codesign": {}, + hostDetailQueryPrefix + "google_chrome_profiles": {}, + hostDetailQueryPrefix + "mdm": {}, + hostDetailQueryPrefix + "munki_info": {}, + hostDetailQueryPrefix + "windows_update_history": {}, + hostDetailQueryPrefix + "kubequery_info": {}, + hostDetailQueryPrefix + "orbit_info": {}, + hostDetailQueryPrefix + "software_vscode_extensions": {}, + hostDetailQueryPrefix + "software_python_packages": {}, + hostDetailQueryPrefix + "software_python_packages_with_users_dir": {}, + hostDetailQueryPrefix + "software_macos_firefox": {}, + hostDetailQueryPrefix + "battery": {}, + hostDetailQueryPrefix + "software_macos_codesign": {}, } for name := range queries { require.NotEmpty(t, discovery[name]) @@ -3709,6 +3711,25 @@ func TestPreProcessSoftwareResults(t *testing.T) { "installed_path": "/some/override/path", } + pythonPackageOne := map[string]string{ + "name": "cryptography", + "version": "41.0.7", + "extension_id": "", + "browser": "", + "source": "python_packages", + "vendor": "", + "installed_path": "/usr/lib/python3/dist-packages", + } + pythonPackageTwo := map[string]string{ + "name": "pip", + "version": "25.0.1", + "extension_id": "", + "browser": "", + "source": "python_packages", + "vendor": "", + "installed_path": "/Users/fleetdm/.pyenv/versions/3.13.1/lib/python3.13/site-packages", + } + for _, tc := range []struct { name string host *fleet.Host @@ -3719,6 +3740,50 @@ func TestPreProcessSoftwareResults(t *testing.T) { resultsOut fleet.OsqueryDistributedQueryResults }{ + { + name: "python packages using original query in extras adds results", + + statusesIn: map[string]fleet.OsqueryStatus{ + hostDetailQueryPrefix + "software_macos": fleet.StatusOK, + hostDetailQueryPrefix + "software_python_packages": fleet.StatusOK, + }, + resultsIn: fleet.OsqueryDistributedQueryResults{ + hostDetailQueryPrefix + "software_macos": []map[string]string{ + foobarApp, + }, + hostDetailQueryPrefix + "software_python_packages": []map[string]string{ + pythonPackageOne, + }, + }, + resultsOut: fleet.OsqueryDistributedQueryResults{ + hostDetailQueryPrefix + "software_macos": []map[string]string{ + foobarApp, + pythonPackageOne, + }, + }, + }, + { + name: "python packages using user query in extras adds results", + + statusesIn: map[string]fleet.OsqueryStatus{ + hostDetailQueryPrefix + "software_macos": fleet.StatusOK, + hostDetailQueryPrefix + "software_python_packages_with_users_dir": fleet.StatusOK, + }, + resultsIn: fleet.OsqueryDistributedQueryResults{ + hostDetailQueryPrefix + "software_macos": []map[string]string{ + foobarApp, + }, + hostDetailQueryPrefix + "software_python_packages_with_users_dir": []map[string]string{ + pythonPackageTwo, + }, + }, + resultsOut: fleet.OsqueryDistributedQueryResults{ + hostDetailQueryPrefix + "software_macos": []map[string]string{ + foobarApp, + pythonPackageTwo, + }, + }, + }, { name: "software query works and there are vs code extensions in extra", diff --git a/server/service/osquery_utils/queries.go b/server/service/osquery_utils/queries.go index 5eebde8067..3bbae325b9 100644 --- a/server/service/osquery_utils/queries.go +++ b/server/service/osquery_utils/queries.go @@ -840,18 +840,6 @@ SELECT path AS installed_path FROM apps UNION -SELECT - name AS name, - version AS version, - '' AS bundle_identifier, - '' AS extension_id, - '' AS browser, - 'python_packages' AS source, - '' AS vendor, - 0 AS last_opened_at, - path AS installed_path -FROM python_packages -UNION SELECT name AS name, version AS version, @@ -1024,19 +1012,7 @@ SELECT '' AS vendor, '' AS arch, path AS installed_path -FROM cached_users CROSS JOIN firefox_addons USING (uid) -UNION -SELECT - name AS name, - version AS version, - '' AS extension_id, - '' AS browser, - 'python_packages' AS source, - '' AS release, - '' AS vendor, - '' AS arch, - path AS installed_path -FROM python_packages; +FROM cached_users CROSS JOIN firefox_addons USING (uid); `), Platforms: fleet.HostLinuxOSs, DirectIngestFunc: directIngestSoftware, @@ -1054,16 +1030,6 @@ SELECT install_location AS installed_path FROM programs UNION -SELECT - name AS name, - version AS version, - '' AS extension_id, - '' AS browser, - 'python_packages' AS source, - '' AS vendor, - path AS installed_path -FROM python_packages -UNION SELECT name AS name, version AS version, @@ -1108,6 +1074,44 @@ FROM chocolatey_packages DirectIngestFunc: directIngestSoftware, } +// In osquery versions < 5.16.0 use the original python_packages query, as the cross join on +// users is not supported +var softwarePythonPackages = DetailQuery{ + Description: "Prior to osquery version 5.16.0, the python_packages table did not search user directories.", + Query: ` + SELECT + name AS name, + version AS version, + '' AS extension_id, + '' AS browser, + 'python_packages' AS source, + '' AS vendor, + path AS installed_path + FROM python_packages + `, + Platforms: append(fleet.HostLinuxOSs, "darwin", "windows"), + Discovery: `SELECT 1 FROM osquery_info WHERE version_compare(version, '5.16.0') < 0`, +} + +// In osquery versions >= 5.16.0 the python_packages table was modified to allow for a +// cross join on users so that user directories could be searched for python packages +var softwarePythonPackagesWithUsersDir = DetailQuery{ + Description: "As of osquery version 5.16.0, the python_packages table searches user directories with support from a cross join on users. See https://fleetdm.com/guides/osquery-consider-joining-against-the-users-table.", + Query: withCachedUsers(`WITH cached_users AS (%s) + SELECT + name AS name, + version AS version, + '' AS extension_id, + '' AS browser, + 'python_packages' AS source, + '' AS vendor, + path AS installed_path + FROM cached_users CROSS JOIN python_packages USING (uid) + `), + Platforms: append(fleet.HostLinuxOSs, "darwin", "windows"), + Discovery: `SELECT 1 FROM osquery_info WHERE version_compare(version, '5.16.0') >= 0`, +} + var softwareChrome = DetailQuery{ Query: `SELECT name AS name, @@ -2244,6 +2248,8 @@ func GetDetailQueries( generatedMap["software_linux"] = softwareLinux generatedMap["software_windows"] = softwareWindows generatedMap["software_chrome"] = softwareChrome + generatedMap["software_python_packages"] = softwarePythonPackages + generatedMap["software_python_packages_with_users_dir"] = softwarePythonPackagesWithUsersDir generatedMap["software_vscode_extensions"] = softwareVSCodeExtensions for key, query := range SoftwareOverrideQueries { diff --git a/server/service/osquery_utils/queries_test.go b/server/service/osquery_utils/queries_test.go index e461b2033d..1a74e3a267 100644 --- a/server/service/osquery_utils/queries_test.go +++ b/server/service/osquery_utils/queries_test.go @@ -307,7 +307,7 @@ func TestGetDetailQueries(t *testing.T) { queriesWithUsersAndSoftware := GetDetailQueries(context.Background(), config.FleetConfig{App: config.AppConfig{EnableScheduledQueryStats: true}}, nil, &fleet.Features{EnableHostUsers: true, EnableSoftwareInventory: true}) qs = baseQueries qs = append(qs, "users", "users_chrome", "software_macos", "software_linux", "software_windows", "software_vscode_extensions", - "software_chrome", "scheduled_query_stats", "software_macos_firefox", "software_macos_codesign") + "software_chrome", "software_python_packages", "software_python_packages_with_users_dir", "scheduled_query_stats", "software_macos_firefox", "software_macos_codesign") require.Len(t, queriesWithUsersAndSoftware, len(qs)) sortedKeysCompare(t, queriesWithUsersAndSoftware, qs)