Issue 1588 allow disabling host users (#1611)

* Allow users to disable host_users

* Add missing files

* Address review comments

* Fix tests

* Check additional queries for nil, not the whole hostsettings

* Finally fix tests
This commit is contained in:
Tomas Touceda
2021-08-11 14:56:11 -03:00
committed by GitHub
parent c781058617
commit 11dff7ca92
16 changed files with 1112 additions and 871 deletions
@@ -0,0 +1 @@
* In some cases, host users are not useful. Users can disable this through the enable_host_users config.
+58
View File
@@ -174,3 +174,61 @@ spec:
assert.Equal(t, &newAgentOpts, teamsByName["team1"].AgentOptions)
assert.Equal(t, []*fleet.EnrollSecret{{Secret: "AAA"}}, enrolledSecretsCalled[uint(42)])
}
func writeTmpYml(t *testing.T, contents string) string {
tmpFile, err := ioutil.TempFile(t.TempDir(), "*.yml")
require.NoError(t, err)
_, err = tmpFile.WriteString(contents)
require.NoError(t, err)
return tmpFile.Name()
}
func TestApplyAppConfig(t *testing.T) {
server, ds := runServerWithMockedDS(t)
defer server.Close()
ds.ListUsersFunc = func(opt fleet.UserListOptions) ([]*fleet.User, error) {
return userRoleSpecList, nil
}
ds.UserByEmailFunc = func(email string) (*fleet.User, error) {
if email == "admin1@example.com" {
return userRoleSpecList[0], nil
}
return userRoleSpecList[1], nil
}
ds.AppConfigFunc = func() (*fleet.AppConfig, error) {
return &fleet.AppConfig{}, nil
}
var savedAppConfig *fleet.AppConfig
ds.SaveAppConfigFunc = func(config *fleet.AppConfig) error {
savedAppConfig = config
return nil
}
name := writeTmpYml(t, `---
apiVersion: v1
kind: config
spec:
host_settings:
enable_host_users: false
`)
assert.Equal(t, "[+] applied fleet config\n", runAppForTest(t, []string{"apply", "-f", name}))
require.NotNil(t, savedAppConfig)
assert.False(t, savedAppConfig.EnableHostUsers)
name = writeTmpYml(t, `---
apiVersion: v1
kind: config
spec:
host_settings:
enable_host_users: true
`)
assert.Equal(t, "[+] applied fleet config\n", runAppForTest(t, []string{"apply", "-f", name}))
require.NotNil(t, savedAppConfig)
assert.True(t, savedAppConfig.EnableHostUsers)
}
+62
View File
@@ -239,3 +239,65 @@ spec:
assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "hosts", "--yaml"}))
assert.Equal(t, expectedJson, runAppForTest(t, []string{"get", "hosts", "--json"}))
}
func TestGetConfig(t *testing.T) {
server, ds := runServerWithMockedDS(t)
defer server.Close()
ds.AppConfigFunc = func() (*fleet.AppConfig, error) {
return &fleet.AppConfig{
EnableHostUsers: true,
VulnerabilityDatabasesPath: ptr.String("/some/path"),
}, nil
}
expectedYaml := `---
apiVersion: v1
kind: config
spec:
agent_options: null
host_expiry_settings:
host_expiry_enabled: false
host_expiry_window: 0
host_settings:
enable_host_users: true
org_info:
org_logo_url: ""
org_name: ""
server_settings:
enable_analytics: false
live_query_disabled: false
server_url: ""
smtp_settings:
authentication_method: authmethod_plain
authentication_type: authtype_username_password
configured: false
domain: ""
enable_smtp: false
enable_ssl_tls: false
enable_start_tls: false
password: '********'
port: 0
sender_address: ""
server: ""
user_name: ""
verify_ssl_certs: false
sso_settings:
enable_sso: false
enable_sso_idp_login: false
entity_id: ""
idp_image_url: ""
idp_name: ""
issuer_uri: ""
metadata: ""
metadata_url: ""
vulnerability_settings:
databases_path: /some/path
`
expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"authtype_username_password","user_name":"","password":"********","enable_ssl_tls":false,"authentication_method":"authmethod_plain","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true},"agent_options":null,"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"}}}
`
assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "config"}))
assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "config", "--yaml"}))
assert.Equal(t, expectedJson, runAppForTest(t, []string{"get", "config", "--json"}))
}
+1
View File
@@ -854,6 +854,7 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs=
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c h1:TWQ2UvXPkhPxI2KmApKBOCaV6yD2N4mlvqFQ/DlPtpQ=
github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c/go.mod h1:OYYulo9tUqRadRLwB0+LE914sa1ui2yL7OrcU3Q/1XY=
+6 -3
View File
@@ -123,9 +123,10 @@ func (d *Datastore) SaveAppConfig(info *fleet.AppConfig) error {
additional_queries,
agent_options,
enable_analytics,
vulnerability_databases_path
vulnerability_databases_path,
enable_host_users
)
VALUES( 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )
VALUES( 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )
ON DUPLICATE KEY UPDATE
org_name = VALUES(org_name),
org_logo_url = VALUES(org_logo_url),
@@ -158,7 +159,8 @@ func (d *Datastore) SaveAppConfig(info *fleet.AppConfig) error {
additional_queries = VALUES(additional_queries),
agent_options = VALUES(agent_options),
enable_analytics = VALUES(enable_analytics),
vulnerability_databases_path = VALUES(vulnerability_databases_path)
vulnerability_databases_path = VALUES(vulnerability_databases_path),
enable_host_users = VALUES(enable_host_users)
`
_, err = tx.Exec(insertStatement,
@@ -194,6 +196,7 @@ func (d *Datastore) SaveAppConfig(info *fleet.AppConfig) error {
info.AgentOptions,
info.EnableAnalytics,
info.VulnerabilityDatabasesPath,
info.EnableHostUsers,
)
if err != nil {
return err
@@ -0,0 +1,22 @@
package tables
import (
"database/sql"
"github.com/pkg/errors"
)
func init() {
MigrationClient.AddMigration(Up_20210810095603, Down_20210810095603)
}
func Up_20210810095603(tx *sql.Tx) error {
if _, err := tx.Exec(`ALTER TABLE app_configs ADD COLUMN enable_host_users BOOL DEFAULT TRUE`); err != nil {
return errors.Wrap(err, "add column enable_host_users")
}
return nil
}
func Down_20210810095603(tx *sql.Tx) error {
return nil
}
+6 -4
View File
@@ -186,6 +186,9 @@ type AppConfig struct {
// VulnerabilityDatabases path
VulnerabilityDatabasesPath *string `db:"vulnerability_databases_path"`
// EnableHostUsers indicates whether the users of each host will be queried and stored
EnableHostUsers bool `db:"enable_host_users" json:"enable_host_users"`
}
func (c AppConfig) AuthzType() string {
@@ -309,7 +312,8 @@ type HostExpirySettings struct {
}
type HostSettings struct {
AdditionalQueries *json.RawMessage `json:"additional_queries"`
EnableHostUsers *bool `json:"enable_host_users"`
AdditionalQueries *json.RawMessage `json:"additional_queries,omitempty"`
}
type OrderDirection int
@@ -398,7 +402,7 @@ type Logging struct {
}
type LoggingPlugin struct {
Plugin string `json:"plugin"`
Plugin string `json:"plugin"`
Config interface{} `json:"config"`
}
@@ -430,5 +434,3 @@ type LambdaConfig struct {
StatusFunction string `json:"status_function"`
ResultFunction string `json:"result_function"`
}
+10 -12
View File
@@ -27,8 +27,8 @@ type appConfigResponse struct {
VulnerabilitySettings *fleet.VulnerabilitySettingsPayload `json:"vulnerability_settings"`
// Logging is loaded on the fly rather than from the database.
Logging *fleet.Logging `json:"logging,omitempty"`
Err error `json:"error,omitempty"`
Logging *fleet.Logging `json:"logging,omitempty"`
Err error `json:"error,omitempty"`
}
func (r appConfigResponse) error() error { return r.Err }
@@ -85,11 +85,9 @@ func makeGetAppConfigEndpoint(svc fleet.Service) endpoint.Endpoint {
}
}
hostSettings := &fleet.HostSettings{
EnableHostUsers: &config.EnableHostUsers,
AdditionalQueries: config.AdditionalQueries,
}
if config.AdditionalQueries == nil {
hostSettings = nil
}
response := appConfigResponse{
OrgInfo: &fleet.OrgInfo{
OrgName: &config.OrgName,
@@ -100,13 +98,13 @@ func makeGetAppConfigEndpoint(svc fleet.Service) endpoint.Endpoint {
LiveQueryDisabled: &config.LiveQueryDisabled,
EnableAnalytics: &config.EnableAnalytics,
},
SMTPSettings: smtpSettings,
SSOSettings: ssoSettings,
HostExpirySettings: hostExpirySettings,
HostSettings: hostSettings,
License: license,
AgentOptions: agentOptions,
Logging: loggingConfig,
SMTPSettings: smtpSettings,
SSOSettings: ssoSettings,
HostExpirySettings: hostExpirySettings,
HostSettings: hostSettings,
License: license,
AgentOptions: agentOptions,
Logging: loggingConfig,
VulnerabilitySettings: vulnerabilitySettings,
}
return response, nil
+3 -1
View File
@@ -253,6 +253,7 @@ func TestAppConfigAdditionalQueriesCanBeRemoved(t *testing.T) {
host_settings:
additional_queries:
time: SELECT * FROM time
enable_host_users: true
`)
applyConfig(t, spec, server, token)
@@ -261,11 +262,12 @@ func TestAppConfigAdditionalQueriesCanBeRemoved(t *testing.T) {
host_expiry_enabled: false
host_expiry_window: 0
host_settings:
enable_host_users: true
`)
applyConfig(t, spec, server, token)
config := getConfig(t, server, token)
assert.Nil(t, config.HostSettings)
assert.Nil(t, config.HostSettings.AdditionalQueries)
}
func TestAppConfigHasLogging(t *testing.T) {
+576
View File
@@ -0,0 +1,576 @@
package osquery_utils
import (
"fmt"
"net"
"os"
"strconv"
"strings"
"time"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/pkg/errors"
"github.com/spf13/cast"
)
type DetailQuery struct {
Query string
// Platforms is a list of platforms to run the query on. If this value is
// empty, run on all platforms.
Platforms []string
IngestFunc func(logger log.Logger, host *fleet.Host, rows []map[string]string) error
}
// RunsForPlatform determines whether this detail query should run on the given platform
func (q *DetailQuery) RunsForPlatform(platform string) bool {
if len(q.Platforms) == 0 {
return true
}
for _, p := range q.Platforms {
if p == platform {
return true
}
}
return false
}
// detailQueries defines the detail queries that should be run on the host, as
// well as how the results of those queries should be ingested into the
// fleet.Host data model. This map should not be modified at runtime.
var detailQueries = map[string]DetailQuery{
"network_interface": {
Query: `select address, mac
from interface_details id join interface_addresses ia
on ia.interface = id.interface where length(mac) > 0
order by (ibytes + obytes) desc`,
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) (err error) {
if len(rows) == 0 {
logger.Log("component", "service", "method", "IngestFunc", "err",
"detail_query_network_interface expected 1 or more results")
return nil
}
// Rows are ordered by traffic, so we will get the most active
// interface by iterating in order
var firstIPv4, firstIPv6 map[string]string
for _, row := range rows {
ip := net.ParseIP(row["address"])
if ip == nil {
continue
}
// Skip link-local and loopback interfaces
if ip.IsLinkLocalUnicast() || ip.IsLoopback() {
continue
}
if strings.Contains(row["address"], ":") {
//IPv6
if firstIPv6 == nil {
firstIPv6 = row
}
} else {
// IPv4
if firstIPv4 == nil {
firstIPv4 = row
}
}
}
var selected map[string]string
switch {
// Prefer IPv4
case firstIPv4 != nil:
selected = firstIPv4
// Otherwise IPv6
case firstIPv6 != nil:
selected = firstIPv6
// If only link-local and loopback found, still use the first
// interface so that we don't get an empty value.
default:
selected = rows[0]
}
host.PrimaryIP = selected["address"]
host.PrimaryMac = selected["mac"]
return nil
},
},
"os_version": {
Query: "select * from os_version limit 1",
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
if len(rows) != 1 {
logger.Log("component", "service", "method", "IngestFunc", "err",
fmt.Sprintf("detail_query_os_version expected single result got %d", len(rows)))
return nil
}
host.OSVersion = fmt.Sprintf(
"%s %s.%s.%s",
rows[0]["name"],
rows[0]["major"],
rows[0]["minor"],
rows[0]["patch"],
)
host.OSVersion = strings.Trim(host.OSVersion, ".")
if build, ok := rows[0]["build"]; ok {
host.Build = build
}
host.Platform = rows[0]["platform"]
host.PlatformLike = rows[0]["platform_like"]
host.CodeName = rows[0]["code_name"]
// On centos6 there is an osquery bug that leaves
// platform empty. Here we workaround.
if host.Platform == "" &&
strings.Contains(strings.ToLower(rows[0]["name"]), "centos") {
host.Platform = "centos"
}
return nil
},
},
"osquery_flags": {
// Collect the interval info (used for online status
// calculation) from the osquery flags. We typically control
// distributed_interval (but it's not required), and typically
// do not control config_tls_refresh.
Query: `select name, value from osquery_flags where name in ("distributed_interval", "config_tls_refresh", "config_refresh", "logger_tls_period")`,
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
var configTLSRefresh, configRefresh uint
var configRefreshSeen, configTLSRefreshSeen bool
for _, row := range rows {
switch row["name"] {
case "distributed_interval":
interval, err := strconv.Atoi(EmptyToZero(row["value"]))
if err != nil {
return errors.Wrap(err, "parsing distributed_interval")
}
host.DistributedInterval = uint(interval)
case "config_tls_refresh":
// Prior to osquery 2.4.6, the flag was
// called `config_tls_refresh`.
interval, err := strconv.Atoi(EmptyToZero(row["value"]))
if err != nil {
return errors.Wrap(err, "parsing config_tls_refresh")
}
configTLSRefresh = uint(interval)
configTLSRefreshSeen = true
case "config_refresh":
// After 2.4.6 `config_tls_refresh` was
// aliased to `config_refresh`.
interval, err := strconv.Atoi(EmptyToZero(row["value"]))
if err != nil {
return errors.Wrap(err, "parsing config_refresh")
}
configRefresh = uint(interval)
configRefreshSeen = true
case "logger_tls_period":
interval, err := strconv.Atoi(EmptyToZero(row["value"]))
if err != nil {
return errors.Wrap(err, "parsing logger_tls_period")
}
host.LoggerTLSPeriod = uint(interval)
}
}
// Since the `config_refresh` flag existed prior to
// 2.4.6 and had a different meaning, we prefer
// `config_tls_refresh` if it was set, and use
// `config_refresh` as a fallback.
if configTLSRefreshSeen {
host.ConfigTLSRefresh = configTLSRefresh
} else if configRefreshSeen {
host.ConfigTLSRefresh = configRefresh
}
return nil
},
},
"osquery_info": {
Query: "select * from osquery_info limit 1",
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
if len(rows) != 1 {
logger.Log("component", "service", "method", "IngestFunc", "err",
fmt.Sprintf("detail_query_osquery_info expected single result got %d", len(rows)))
return nil
}
host.OsqueryVersion = rows[0]["version"]
return nil
},
},
"system_info": {
Query: "select * from system_info limit 1",
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
if len(rows) != 1 {
logger.Log("component", "service", "method", "IngestFunc", "err",
fmt.Sprintf("detail_query_system_info expected single result got %d", len(rows)))
return nil
}
var err error
host.Memory, err = strconv.ParseInt(EmptyToZero(rows[0]["physical_memory"]), 10, 64)
if err != nil {
return err
}
host.Hostname = rows[0]["hostname"]
host.UUID = rows[0]["uuid"]
host.CPUType = rows[0]["cpu_type"]
host.CPUSubtype = rows[0]["cpu_subtype"]
host.CPUBrand = rows[0]["cpu_brand"]
host.CPUPhysicalCores, err = strconv.Atoi(EmptyToZero(rows[0]["cpu_physical_cores"]))
if err != nil {
return err
}
host.CPULogicalCores, err = strconv.Atoi(EmptyToZero(rows[0]["cpu_logical_cores"]))
if err != nil {
return err
}
host.HardwareVendor = rows[0]["hardware_vendor"]
host.HardwareModel = rows[0]["hardware_model"]
host.HardwareVersion = rows[0]["hardware_version"]
host.HardwareSerial = rows[0]["hardware_serial"]
host.ComputerName = rows[0]["computer_name"]
return nil
},
},
"uptime": {
Query: "select * from uptime limit 1",
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
if len(rows) != 1 {
logger.Log("component", "service", "method", "IngestFunc", "err",
fmt.Sprintf("detail_query_uptime expected single result got %d", len(rows)))
return nil
}
uptimeSeconds, err := strconv.Atoi(EmptyToZero(rows[0]["total_seconds"]))
if err != nil {
return err
}
host.Uptime = time.Duration(uptimeSeconds) * time.Second
return nil
},
},
"scheduled_query_stats": {
Query: `
SELECT *,
(SELECT value from osquery_flags where name = 'pack_delimiter') AS delimiter
FROM osquery_schedule
`,
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
packs := map[string][]fleet.ScheduledQueryStats{}
for _, row := range rows {
providedName := row["name"]
if providedName == "" {
level.Debug(logger).Log(
"msg", "host reported scheduled query with empty name",
"host", host.Hostname,
)
continue
}
delimiter := row["delimiter"]
if delimiter == "" {
level.Debug(logger).Log(
"msg", "host reported scheduled query with empty delimiter",
"host", host.Hostname,
)
continue
}
// Split with a limit of 2 in case query name includes the
// delimiter. Not much we can do if pack name includes the
// delimiter.
trimmedName := strings.TrimPrefix(providedName, "pack"+delimiter)
parts := strings.SplitN(trimmedName, delimiter, 2)
if len(parts) != 2 {
level.Debug(logger).Log(
"msg", "could not split pack and query names",
"host", host.Hostname,
"name", providedName,
"delimiter", delimiter,
)
continue
}
packName, scheduledName := parts[0], parts[1]
stats := fleet.ScheduledQueryStats{
ScheduledQueryName: scheduledName,
PackName: packName,
AverageMemory: cast.ToInt(row["average_memory"]),
Denylisted: cast.ToBool(row["denylisted"]),
Executions: cast.ToInt(row["executions"]),
Interval: cast.ToInt(row["interval"]),
// Cast to int first to allow cast.ToTime to interpret the unix timestamp.
LastExecuted: time.Unix(cast.ToInt64(row["last_executed"]), 0).UTC(),
OutputSize: cast.ToInt(row["output_size"]),
SystemTime: cast.ToInt(row["system_time"]),
UserTime: cast.ToInt(row["user_time"]),
WallTime: cast.ToInt(row["wall_time"]),
}
packs[packName] = append(packs[packName], stats)
}
host.PackStats = []fleet.PackStats{}
for packName, stats := range packs {
host.PackStats = append(
host.PackStats,
fleet.PackStats{
PackName: packName,
QueryStats: stats,
},
)
}
return nil
},
},
}
var softwareMacOS = DetailQuery{
Query: `
SELECT
name AS name,
bundle_short_version AS version,
'Application (macOS)' AS type,
'apps' AS source
FROM apps
UNION
SELECT
name AS name,
version AS version,
'Package (Python)' AS type,
'python_packages' AS source
FROM python_packages
UNION
SELECT
name AS name,
version AS version,
'Browser plugin (Chrome)' AS type,
'chrome_extensions' AS source
FROM chrome_extensions
UNION
SELECT
name AS name,
version AS version,
'Browser plugin (Firefox)' AS type,
'firefox_addons' AS source
FROM firefox_addons
UNION
SELECT
name As name,
version AS version,
'Browser plugin (Safari)' AS type,
'safari_extensions' AS source
FROM safari_extensions
UNION
SELECT
name AS name,
version AS version,
'Package (Homebrew)' AS type,
'homebrew_packages' AS source
FROM homebrew_packages;
`,
Platforms: []string{"darwin"},
IngestFunc: ingestSoftware,
}
var softwareLinux = DetailQuery{
Query: `
SELECT
name AS name,
version AS version,
'Package (deb)' AS type,
'deb_packages' AS source
FROM deb_packages
UNION
SELECT
package AS name,
version AS version,
'Package (Portage)' AS type,
'portage_packages' AS source
FROM portage_packages
UNION
SELECT
name AS name,
version AS version,
'Package (RPM)' AS type,
'rpm_packages' AS source
FROM rpm_packages
UNION
SELECT
name AS name,
version AS version,
'Package (NPM)' AS type,
'npm_packages' AS source
FROM npm_packages
UNION
SELECT
name AS name,
version AS version,
'Package (Atom)' AS type,
'atom_packages' AS source
FROM atom_packages
UNION
SELECT
name AS name,
version AS version,
'Package (Python)' AS type,
'python_packages' AS source
FROM python_packages;
`,
Platforms: []string{"linux", "rhel", "ubuntu", "centos"},
IngestFunc: ingestSoftware,
}
var softwareWindows = DetailQuery{
Query: `
SELECT
name AS name,
version AS version,
'Program (Windows)' AS type,
'programs' AS source
FROM programs
UNION
SELECT
name AS name,
version AS version,
'Package (Python)' AS type,
'python_packages' AS source
FROM python_packages
UNION
SELECT
name AS name,
version AS version,
'Browser plugin (IE)' AS type,
'ie_extensions' AS source
FROM ie_extensions
UNION
SELECT
name AS name,
version AS version,
'Browser plugin (Chrome)' AS type,
'chrome_extensions' AS source
FROM chrome_extensions
UNION
SELECT
name AS name,
version AS version,
'Browser plugin (Firefox)' AS type,
'firefox_addons' AS source
FROM firefox_addons
UNION
SELECT
name AS name,
version AS version,
'Package (Chocolatey)' AS type,
'chocolatey_packages' AS source
FROM chocolatey_packages
UNION
SELECT
name AS name,
version AS version,
'Package (Atom)' AS type,
'atom_packages' AS source
FROM atom_packages
UNION
SELECT
name AS name,
version AS version,
'Package (Python)' AS type,
'python_packages' AS source
FROM python_packages;
`,
Platforms: []string{"windows"},
IngestFunc: ingestSoftware,
}
var usersQuery = DetailQuery{
Query: `SELECT uid, username, type, groupname FROM users u JOIN groups g ON g.gid=u.gid;`,
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
var users []fleet.HostUser
for _, row := range rows {
uid, err := strconv.Atoi(row["uid"])
if err != nil {
return errors.Wrapf(err, "converting uid %s to int", row["uid"])
}
username := row["username"]
type_ := row["type"]
groupname := row["groupname"]
u := fleet.HostUser{
Uid: uint(uid),
Username: username,
Type: type_,
GroupName: groupname,
}
users = append(users, u)
}
host.Users = users
return nil
},
}
func ingestSoftware(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
software := fleet.HostSoftware{Modified: true}
for _, row := range rows {
name := row["name"]
version := row["version"]
source := row["source"]
if name == "" {
level.Debug(logger).Log(
"msg", "host reported software with empty name",
"host", host.Hostname,
"version", version,
"source", source,
)
continue
}
if source == "" {
level.Debug(logger).Log(
"msg", "host reported software with empty name",
"host", host.Hostname,
"version", version,
"name", name,
)
continue
}
s := fleet.Software{Name: name, Version: version, Source: source}
software.Software = append(software.Software, s)
}
host.HostSoftware = software
return nil
}
func GetDetailQueries(ac *fleet.AppConfig) map[string]DetailQuery {
generatedMap := make(map[string]DetailQuery)
for key, query := range detailQueries {
generatedMap[key] = query
}
if os.Getenv("FLEET_BETA_SOFTWARE_INVENTORY") != "" {
generatedMap["software_macos"] = softwareMacOS
generatedMap["software_linux"] = softwareLinux
generatedMap["software_windows"] = softwareWindows
}
if ac != nil && ac.EnableHostUsers {
generatedMap["users"] = usersQuery
}
return generatedMap
}
@@ -0,0 +1,303 @@
package osquery_utils
import (
"encoding/json"
"os"
"sort"
"testing"
"time"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/go-kit/kit/log"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDetailQueryNetworkInterfaces(t *testing.T) {
var initialHost fleet.Host
host := initialHost
ingest := GetDetailQueries(nil)["network_interface"].IngestFunc
assert.NoError(t, ingest(log.NewNopLogger(), &host, nil))
assert.Equal(t, initialHost, host)
var rows []map[string]string
require.NoError(t, json.Unmarshal([]byte(`
[
{"address":"127.0.0.1","mac":"00:00:00:00:00:00"},
{"address":"::1","mac":"00:00:00:00:00:00"},
{"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"},
{"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"},
{"address":"192.168.1.3","mac":"f4:5d:79:93:58:5b"},
{"address":"fe80::241a:9aff:fe60:d80a%awdl0","mac":"27:1b:aa:60:e8:0a"},
{"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"}
]`),
&rows,
))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.Equal(t, "192.168.1.3", host.PrimaryIP)
assert.Equal(t, "f4:5d:79:93:58:5b", host.PrimaryMac)
// Only IPv6
require.NoError(t, json.Unmarshal([]byte(`
[
{"address":"127.0.0.1","mac":"00:00:00:00:00:00"},
{"address":"::1","mac":"00:00:00:00:00:00"},
{"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"},
{"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"},
{"address":"2604:3f08:1337:9411:cbe:814f:51a6:e4e3","mac":"27:1b:aa:60:e8:0a"},
{"address":"3333:3f08:1337:9411:cbe:814f:51a6:e4e3","mac":"bb:1b:aa:60:e8:bb"},
{"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"}
]`),
&rows,
))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.Equal(t, "2604:3f08:1337:9411:cbe:814f:51a6:e4e3", host.PrimaryIP)
assert.Equal(t, "27:1b:aa:60:e8:0a", host.PrimaryMac)
// IPv6 appears before IPv4 (v4 should be prioritized)
require.NoError(t, json.Unmarshal([]byte(`
[
{"address":"127.0.0.1","mac":"00:00:00:00:00:00"},
{"address":"::1","mac":"00:00:00:00:00:00"},
{"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"},
{"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"},
{"address":"2604:3f08:1337:9411:cbe:814f:51a6:e4e3","mac":"27:1b:aa:60:e8:0a"},
{"address":"205.111.43.79","mac":"ab:1b:aa:60:e8:0a"},
{"address":"205.111.44.80","mac":"bb:bb:aa:60:e8:0a"},
{"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"}
]`),
&rows,
))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.Equal(t, "205.111.43.79", host.PrimaryIP)
assert.Equal(t, "ab:1b:aa:60:e8:0a", host.PrimaryMac)
// Only link-local/loopback
require.NoError(t, json.Unmarshal([]byte(`
[
{"address":"127.0.0.1","mac":"00:00:00:00:00:00"},
{"address":"::1","mac":"00:00:00:00:00:00"},
{"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"},
{"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"},
{"address":"fe80::241a:9aff:fe60:d80a%awdl0","mac":"27:1b:aa:60:e8:0a"},
{"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"}
]`),
&rows,
))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.Equal(t, "127.0.0.1", host.PrimaryIP)
assert.Equal(t, "00:00:00:00:00:00", host.PrimaryMac)
}
func TestDetailQueryScheduledQueryStats(t *testing.T) {
host := fleet.Host{}
ingest := GetDetailQueries(nil)["scheduled_query_stats"].IngestFunc
assert.NoError(t, ingest(log.NewNopLogger(), &host, nil))
assert.Len(t, host.PackStats, 0)
resJSON := `
[
{
"average_memory":"33",
"delimiter":"/",
"denylisted":"0",
"executions":"1",
"interval":"33",
"last_executed":"1620325191",
"name":"pack/pack-2/time",
"output_size":"",
"query":"SELECT * FROM time",
"system_time":"100",
"user_time":"60",
"wall_time":"180"
},
{
"average_memory":"8000",
"delimiter":"/",
"denylisted":"0",
"executions":"164",
"interval":"30",
"last_executed":"1620325191",
"name":"pack/test/osquery info",
"output_size":"1337",
"query":"SELECT * FROM osquery_info",
"system_time":"150",
"user_time":"180",
"wall_time":"0"
},
{
"average_memory":"50400",
"delimiter":"/",
"denylisted":"1",
"executions":"188",
"interval":"30",
"last_executed":"1620325203",
"name":"pack/test/processes?",
"output_size":"",
"query":"SELECT * FROM processes",
"system_time":"140",
"user_time":"190",
"wall_time":"1"
},
{
"average_memory":"0",
"delimiter":"/",
"denylisted":"0",
"executions":"1",
"interval":"3600",
"last_executed":"1620323381",
"name":"pack/test/processes?-1",
"output_size":"",
"query":"SELECT * FROM processes",
"system_time":"0",
"user_time":"0",
"wall_time":"0"
},
{
"average_memory":"0",
"delimiter":"/",
"denylisted":"0",
"executions":"105",
"interval":"47",
"last_executed":"1620325190",
"name":"pack/test/time",
"output_size":"",
"query":"SELECT * FROM time",
"system_time":"70",
"user_time":"50",
"wall_time":"1"
}
]
`
var rows []map[string]string
require.NoError(t, json.Unmarshal([]byte(resJSON), &rows))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.Len(t, host.PackStats, 2)
sort.Slice(host.PackStats, func(i, j int) bool {
return host.PackStats[i].PackName < host.PackStats[j].PackName
})
assert.Equal(t, host.PackStats[0].PackName, "pack-2")
assert.ElementsMatch(t, host.PackStats[0].QueryStats,
[]fleet.ScheduledQueryStats{
{
ScheduledQueryName: "time",
PackName: "pack-2",
AverageMemory: 33,
Denylisted: false,
Executions: 1,
Interval: 33,
LastExecuted: time.Unix(1620325191, 0).UTC(),
OutputSize: 0,
SystemTime: 100,
UserTime: 60,
WallTime: 180,
},
},
)
assert.Equal(t, host.PackStats[1].PackName, "test")
assert.ElementsMatch(t, host.PackStats[1].QueryStats,
[]fleet.ScheduledQueryStats{
{
ScheduledQueryName: "osquery info",
PackName: "test",
AverageMemory: 8000,
Denylisted: false,
Executions: 164,
Interval: 30,
LastExecuted: time.Unix(1620325191, 0).UTC(),
OutputSize: 1337,
SystemTime: 150,
UserTime: 180,
WallTime: 0,
},
{
ScheduledQueryName: "processes?",
PackName: "test",
AverageMemory: 50400,
Denylisted: true,
Executions: 188,
Interval: 30,
LastExecuted: time.Unix(1620325203, 0).UTC(),
OutputSize: 0,
SystemTime: 140,
UserTime: 190,
WallTime: 1,
},
{
ScheduledQueryName: "processes?-1",
PackName: "test",
AverageMemory: 0,
Denylisted: false,
Executions: 1,
Interval: 3600,
LastExecuted: time.Unix(1620323381, 0).UTC(),
OutputSize: 0,
SystemTime: 0,
UserTime: 0,
WallTime: 0,
},
{
ScheduledQueryName: "time",
PackName: "test",
AverageMemory: 0,
Denylisted: false,
Executions: 105,
Interval: 47,
LastExecuted: time.Unix(1620325190, 0).UTC(),
OutputSize: 0,
SystemTime: 70,
UserTime: 50,
WallTime: 1,
},
},
)
assert.NoError(t, ingest(log.NewNopLogger(), &host, nil))
assert.Len(t, host.PackStats, 0)
}
func sortedKeysCompare(t *testing.T, m map[string]DetailQuery, expectedKeys []string) {
var keys []string
for key := range m {
keys = append(keys, key)
}
assert.ElementsMatch(t, keys, expectedKeys)
}
func TestGetDetailQueries(t *testing.T) {
queriesNoConfig := GetDetailQueries(nil)
require.Len(t, queriesNoConfig, 7)
baseQueries := []string{
"network_interface",
"os_version",
"osquery_flags",
"osquery_info",
"scheduled_query_stats",
"system_info",
"uptime",
}
sortedKeysCompare(t, queriesNoConfig, baseQueries)
queriesWithUsers := GetDetailQueries(&fleet.AppConfig{EnableHostUsers: true})
require.Len(t, queriesWithUsers, 8)
sortedKeysCompare(t, queriesWithUsers, append(baseQueries, "users"))
require.NoError(t, os.Setenv("FLEET_BETA_SOFTWARE_INVENTORY", "1"))
queriesWithUsersAndSoftware := GetDetailQueries(&fleet.AppConfig{EnableHostUsers: true})
require.Len(t, queriesWithUsersAndSoftware, 11)
sortedKeysCompare(t, queriesWithUsersAndSoftware,
append(baseQueries, "users", "software_macos", "software_linux", "software_windows"))
require.NoError(t, os.Setenv("FLEET_BETA_SOFTWARE_INVENTORY", ""))
}
+11
View File
@@ -0,0 +1,11 @@
package osquery_utils
// EmptyToZero Sometimes osquery gives us empty string where we expect an integer.
// We change the to "0" so it can be handled by the appropriate string to
// integer conversion function, as these will err on ""
func EmptyToZero(val string) string {
if val == "" {
return "0"
}
return val
}
+3 -2
View File
@@ -190,8 +190,9 @@ func appConfigFromAppConfigPayload(p fleet.AppConfigPayload, config fleet.AppCon
}
if settings := p.HostSettings; settings != nil {
if settings.AdditionalQueries != nil {
config.AdditionalQueries = settings.AdditionalQueries
config.AdditionalQueries = settings.AdditionalQueries
if settings.EnableHostUsers != nil {
config.EnableHostUsers = *settings.EnableHostUsers
}
} else {
config.AdditionalQueries = nil
+34 -567
View File
@@ -4,14 +4,13 @@ import (
"context"
"encoding/json"
"fmt"
"net"
"os"
"strconv"
"strings"
"time"
"github.com/fleetdm/fleet/v4/server"
"github.com/fleetdm/fleet/v4/server/contexts/logging"
"github.com/fleetdm/fleet/v4/server/service/osquery_utils"
kithttp "github.com/go-kit/kit/transport/http"
"github.com/fleetdm/fleet/v4/server/fleet"
@@ -37,16 +36,6 @@ func (e osqueryError) NodeInvalid() bool {
return e.nodeInvalid
}
// Sometimes osquery gives us empty string where we expect an integer.
// We change the to "0" so it can be handled by the appropriate string to
// integer conversion function, as these will err on ""
func emptyToZero(val string) string {
if val == "" {
return "0"
}
return val
}
func (svc Service) AuthenticateHost(ctx context.Context, nodeKey string) (*fleet.Host, error) {
// skipauth: Authorization is currently for user endpoints only.
svc.authz.SkipAuthorization(ctx)
@@ -116,18 +105,32 @@ func (svc Service) EnrollAgent(ctx context.Context, enrollSecret, hostIdentifier
return "", osqueryError{message: "save enroll failed: " + err.Error(), nodeInvalid: true}
}
appConfig, err := svc.ds.AppConfig()
if err != nil {
return "", osqueryError{message: "save enroll failed: " + err.Error(), nodeInvalid: true}
}
// Save enrollment details if provided
detailQueries := osquery_utils.GetDetailQueries(appConfig)
save := false
if r, ok := hostDetails["os_version"]; ok {
detailQueries["os_version"].IngestFunc(svc.logger, host, []map[string]string{r})
err := detailQueries["os_version"].IngestFunc(svc.logger, host, []map[string]string{r})
if err != nil {
return "", errors.Wrap(err, "Ingesting os_version")
}
save = true
}
if r, ok := hostDetails["osquery_info"]; ok {
detailQueries["osquery_info"].IngestFunc(svc.logger, host, []map[string]string{r})
err := detailQueries["osquery_info"].IngestFunc(svc.logger, host, []map[string]string{r})
if err != nil {
return "", errors.Wrap(err, "Ingesting osquery_info")
}
save = true
}
if r, ok := hostDetails["system_info"]; ok {
detailQueries["system_info"].IngestFunc(svc.logger, host, []map[string]string{r})
err := detailQueries["system_info"].IngestFunc(svc.logger, host, []map[string]string{r})
if err != nil {
return "", errors.Wrap(err, "Ingesting system_info")
}
save = true
}
if save {
@@ -372,543 +375,6 @@ const hostAdditionalQueryPrefix = "fleet_additional_query_"
// run from a distributed query campaign
const hostDistributedQueryPrefix = "fleet_distributed_query_"
type detailQuery struct {
Query string
// Platforms is a list of platforms to run the query on. If this value is
// empty, run on all platforms.
Platforms []string
IngestFunc func(logger log.Logger, host *fleet.Host, rows []map[string]string) error
}
// runForPlatform determines whether this detail query should run on the given platform
func (q *detailQuery) runForPlatform(platform string) bool {
if len(q.Platforms) == 0 {
return true
}
for _, p := range q.Platforms {
if p == platform {
return true
}
}
return false
}
// detailQueries defines the detail queries that should be run on the host, as
// well as how the results of those queries should be ingested into the
// fleet.Host data model. This map should not be modified at runtime.
var detailQueries = map[string]detailQuery{
"network_interface": {
Query: `select address, mac
from interface_details id join interface_addresses ia
on ia.interface = id.interface where length(mac) > 0
order by (ibytes + obytes) desc`,
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) (err error) {
if len(rows) == 0 {
logger.Log("component", "service", "method", "IngestFunc", "err",
"detail_query_network_interface expected 1 or more results")
return nil
}
// Rows are ordered by traffic, so we will get the most active
// interface by iterating in order
var firstIPv4, firstIPv6 map[string]string
for _, row := range rows {
ip := net.ParseIP(row["address"])
if ip == nil {
continue
}
// Skip link-local and loopback interfaces
if ip.IsLinkLocalUnicast() || ip.IsLoopback() {
continue
}
if strings.Contains(row["address"], ":") {
//IPv6
if firstIPv6 == nil {
firstIPv6 = row
}
} else {
// IPv4
if firstIPv4 == nil {
firstIPv4 = row
}
}
}
var selected map[string]string
switch {
// Prefer IPv4
case firstIPv4 != nil:
selected = firstIPv4
// Otherwise IPv6
case firstIPv6 != nil:
selected = firstIPv6
// If only link-local and loopback found, still use the first
// interface so that we don't get an empty value.
default:
selected = rows[0]
}
host.PrimaryIP = selected["address"]
host.PrimaryMac = selected["mac"]
return nil
},
},
"os_version": {
Query: "select * from os_version limit 1",
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
if len(rows) != 1 {
logger.Log("component", "service", "method", "IngestFunc", "err",
fmt.Sprintf("detail_query_os_version expected single result got %d", len(rows)))
return nil
}
host.OSVersion = fmt.Sprintf(
"%s %s.%s.%s",
rows[0]["name"],
rows[0]["major"],
rows[0]["minor"],
rows[0]["patch"],
)
host.OSVersion = strings.Trim(host.OSVersion, ".")
if build, ok := rows[0]["build"]; ok {
host.Build = build
}
host.Platform = rows[0]["platform"]
host.PlatformLike = rows[0]["platform_like"]
host.CodeName = rows[0]["code_name"]
// On centos6 there is an osquery bug that leaves
// platform empty. Here we workaround.
if host.Platform == "" &&
strings.Contains(strings.ToLower(rows[0]["name"]), "centos") {
host.Platform = "centos"
}
return nil
},
},
"osquery_flags": {
// Collect the interval info (used for online status
// calculation) from the osquery flags. We typically control
// distributed_interval (but it's not required), and typically
// do not control config_tls_refresh.
Query: `select name, value from osquery_flags where name in ("distributed_interval", "config_tls_refresh", "config_refresh", "logger_tls_period")`,
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
var configTLSRefresh, configRefresh uint
var configRefreshSeen, configTLSRefreshSeen bool
for _, row := range rows {
switch row["name"] {
case "distributed_interval":
interval, err := strconv.Atoi(emptyToZero(row["value"]))
if err != nil {
return errors.Wrap(err, "parsing distributed_interval")
}
host.DistributedInterval = uint(interval)
case "config_tls_refresh":
// Prior to osquery 2.4.6, the flag was
// called `config_tls_refresh`.
interval, err := strconv.Atoi(emptyToZero(row["value"]))
if err != nil {
return errors.Wrap(err, "parsing config_tls_refresh")
}
configTLSRefresh = uint(interval)
configTLSRefreshSeen = true
case "config_refresh":
// After 2.4.6 `config_tls_refresh` was
// aliased to `config_refresh`.
interval, err := strconv.Atoi(emptyToZero(row["value"]))
if err != nil {
return errors.Wrap(err, "parsing config_refresh")
}
configRefresh = uint(interval)
configRefreshSeen = true
case "logger_tls_period":
interval, err := strconv.Atoi(emptyToZero(row["value"]))
if err != nil {
return errors.Wrap(err, "parsing logger_tls_period")
}
host.LoggerTLSPeriod = uint(interval)
}
}
// Since the `config_refresh` flag existed prior to
// 2.4.6 and had a different meaning, we prefer
// `config_tls_refresh` if it was set, and use
// `config_refresh` as a fallback.
if configTLSRefreshSeen {
host.ConfigTLSRefresh = configTLSRefresh
} else if configRefreshSeen {
host.ConfigTLSRefresh = configRefresh
}
return nil
},
},
"osquery_info": {
Query: "select * from osquery_info limit 1",
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
if len(rows) != 1 {
logger.Log("component", "service", "method", "IngestFunc", "err",
fmt.Sprintf("detail_query_osquery_info expected single result got %d", len(rows)))
return nil
}
host.OsqueryVersion = rows[0]["version"]
return nil
},
},
"system_info": {
Query: "select * from system_info limit 1",
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
if len(rows) != 1 {
logger.Log("component", "service", "method", "IngestFunc", "err",
fmt.Sprintf("detail_query_system_info expected single result got %d", len(rows)))
return nil
}
var err error
host.Memory, err = strconv.ParseInt(emptyToZero(rows[0]["physical_memory"]), 10, 64)
if err != nil {
return err
}
host.Hostname = rows[0]["hostname"]
host.UUID = rows[0]["uuid"]
host.CPUType = rows[0]["cpu_type"]
host.CPUSubtype = rows[0]["cpu_subtype"]
host.CPUBrand = rows[0]["cpu_brand"]
host.CPUPhysicalCores, err = strconv.Atoi(emptyToZero(rows[0]["cpu_physical_cores"]))
if err != nil {
return err
}
host.CPULogicalCores, err = strconv.Atoi(emptyToZero(rows[0]["cpu_logical_cores"]))
if err != nil {
return err
}
host.HardwareVendor = rows[0]["hardware_vendor"]
host.HardwareModel = rows[0]["hardware_model"]
host.HardwareVersion = rows[0]["hardware_version"]
host.HardwareSerial = rows[0]["hardware_serial"]
host.ComputerName = rows[0]["computer_name"]
return nil
},
},
"uptime": {
Query: "select * from uptime limit 1",
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
if len(rows) != 1 {
logger.Log("component", "service", "method", "IngestFunc", "err",
fmt.Sprintf("detail_query_uptime expected single result got %d", len(rows)))
return nil
}
uptimeSeconds, err := strconv.Atoi(emptyToZero(rows[0]["total_seconds"]))
if err != nil {
return err
}
host.Uptime = time.Duration(uptimeSeconds) * time.Second
return nil
},
},
"software_macos": {
Query: `
SELECT
name AS name,
bundle_short_version AS version,
'Application (macOS)' AS type,
'apps' AS source
FROM apps
UNION
SELECT
name AS name,
version AS version,
'Package (Python)' AS type,
'python_packages' AS source
FROM python_packages
UNION
SELECT
name AS name,
version AS version,
'Browser plugin (Chrome)' AS type,
'chrome_extensions' AS source
FROM chrome_extensions
UNION
SELECT
name AS name,
version AS version,
'Browser plugin (Firefox)' AS type,
'firefox_addons' AS source
FROM firefox_addons
UNION
SELECT
name As name,
version AS version,
'Browser plugin (Safari)' AS type,
'safari_extensions' AS source
FROM safari_extensions
UNION
SELECT
name AS name,
version AS version,
'Package (Homebrew)' AS type,
'homebrew_packages' AS source
FROM homebrew_packages;
`,
Platforms: []string{"darwin"},
IngestFunc: ingestSoftware,
},
"software_linux": {
Query: `
SELECT
name AS name,
version AS version,
'Package (deb)' AS type,
'deb_packages' AS source
FROM deb_packages
UNION
SELECT
package AS name,
version AS version,
'Package (Portage)' AS type,
'portage_packages' AS source
FROM portage_packages
UNION
SELECT
name AS name,
version AS version,
'Package (RPM)' AS type,
'rpm_packages' AS source
FROM rpm_packages
UNION
SELECT
name AS name,
version AS version,
'Package (NPM)' AS type,
'npm_packages' AS source
FROM npm_packages
UNION
SELECT
name AS name,
version AS version,
'Package (Atom)' AS type,
'atom_packages' AS source
FROM atom_packages
UNION
SELECT
name AS name,
version AS version,
'Package (Python)' AS type,
'python_packages' AS source
FROM python_packages;
`,
Platforms: []string{"linux", "rhel", "ubuntu", "centos"},
IngestFunc: ingestSoftware,
},
"software_windows": {
Query: `
SELECT
name AS name,
version AS version,
'Program (Windows)' AS type,
'programs' AS source
FROM programs
UNION
SELECT
name AS name,
version AS version,
'Package (Python)' AS type,
'python_packages' AS source
FROM python_packages
UNION
SELECT
name AS name,
version AS version,
'Browser plugin (IE)' AS type,
'ie_extensions' AS source
FROM ie_extensions
UNION
SELECT
name AS name,
version AS version,
'Browser plugin (Chrome)' AS type,
'chrome_extensions' AS source
FROM chrome_extensions
UNION
SELECT
name AS name,
version AS version,
'Browser plugin (Firefox)' AS type,
'firefox_addons' AS source
FROM firefox_addons
UNION
SELECT
name AS name,
version AS version,
'Package (Chocolatey)' AS type,
'chocolatey_packages' AS source
FROM chocolatey_packages
UNION
SELECT
name AS name,
version AS version,
'Package (Atom)' AS type,
'atom_packages' AS source
FROM atom_packages
UNION
SELECT
name AS name,
version AS version,
'Package (Python)' AS type,
'python_packages' AS source
FROM python_packages;
`,
Platforms: []string{"windows"},
IngestFunc: ingestSoftware,
},
"scheduled_query_stats": {
Query: `
SELECT *,
(SELECT value from osquery_flags where name = 'pack_delimiter') AS delimiter
FROM osquery_schedule
`,
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
packs := map[string][]fleet.ScheduledQueryStats{}
for _, row := range rows {
providedName := row["name"]
if providedName == "" {
level.Debug(logger).Log(
"msg", "host reported scheduled query with empty name",
"host", host.Hostname,
)
continue
}
delimiter := row["delimiter"]
if delimiter == "" {
level.Debug(logger).Log(
"msg", "host reported scheduled query with empty delimiter",
"host", host.Hostname,
)
continue
}
// Split with a limit of 2 in case query name includes the
// delimiter. Not much we can do if pack name includes the
// delimiter.
trimmedName := strings.TrimPrefix(providedName, "pack"+delimiter)
parts := strings.SplitN(trimmedName, delimiter, 2)
if len(parts) != 2 {
level.Debug(logger).Log(
"msg", "could not split pack and query names",
"host", host.Hostname,
"name", providedName,
"delimiter", delimiter,
)
continue
}
packName, scheduledName := parts[0], parts[1]
stats := fleet.ScheduledQueryStats{
ScheduledQueryName: scheduledName,
PackName: packName,
AverageMemory: cast.ToInt(row["average_memory"]),
Denylisted: cast.ToBool(row["denylisted"]),
Executions: cast.ToInt(row["executions"]),
Interval: cast.ToInt(row["interval"]),
// Cast to int first to allow cast.ToTime to interpret the unix timestamp.
LastExecuted: time.Unix(cast.ToInt64(row["last_executed"]), 0).UTC(),
OutputSize: cast.ToInt(row["output_size"]),
SystemTime: cast.ToInt(row["system_time"]),
UserTime: cast.ToInt(row["user_time"]),
WallTime: cast.ToInt(row["wall_time"]),
}
packs[packName] = append(packs[packName], stats)
}
host.PackStats = []fleet.PackStats{}
for packName, stats := range packs {
host.PackStats = append(
host.PackStats,
fleet.PackStats{
PackName: packName,
QueryStats: stats,
},
)
}
return nil
},
},
"users": {
Query: `SELECT uid, username, type, groupname FROM users u JOIN groups g ON g.gid=u.gid;`,
IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
var users []fleet.HostUser
for _, row := range rows {
uid, err := strconv.Atoi(row["uid"])
if err != nil {
return errors.Wrapf(err, "converting uid %s to int", row["uid"])
}
username := row["username"]
type_ := row["type"]
groupname := row["groupname"]
u := fleet.HostUser{
Uid: uint(uid),
Username: username,
Type: type_,
GroupName: groupname,
}
users = append(users, u)
}
host.Users = users
return nil
},
},
}
func ingestSoftware(logger log.Logger, host *fleet.Host, rows []map[string]string) error {
software := fleet.HostSoftware{Modified: true}
for _, row := range rows {
name := row["name"]
version := row["version"]
source := row["source"]
if name == "" {
level.Debug(logger).Log(
"msg", "host reported software with empty name",
"host", host.Hostname,
"version", version,
"source", source,
)
continue
}
if source == "" {
level.Debug(logger).Log(
"msg", "host reported software with empty name",
"host", host.Hostname,
"version", version,
"name", name,
)
continue
}
s := fleet.Software{Name: name, Version: version, Source: source}
software.Software = append(software.Software, s)
}
host.HostSoftware = software
return nil
}
// hostDetailQueries returns the map of queries that should be executed by
// osqueryd to fill in the host details
func (svc *Service) hostDetailQueries(host fleet.Host) (map[string]string, error) {
@@ -917,25 +383,19 @@ func (svc *Service) hostDetailQueries(host fleet.Host) (map[string]string, error
// No need to update already fresh details
return queries, nil
}
config, err := svc.ds.AppConfig()
if err != nil {
return nil, osqueryError{message: "get additional queries: " + err.Error()}
}
detailQueries := osquery_utils.GetDetailQueries(config)
for name, query := range detailQueries {
if query.runForPlatform(host.Platform) {
if strings.HasPrefix(name, "software_") {
// Feature flag this because of as-yet-untested performance
// considerations.
if os.Getenv("FLEET_BETA_SOFTWARE_INVENTORY") == "" {
continue
}
}
if query.RunsForPlatform(host.Platform) {
queries[hostDetailQueryPrefix+name] = query.Query
}
}
// Get additional queries
config, err := svc.ds.AppConfig()
if err != nil {
return nil, osqueryError{message: "get additional queries: " + err.Error()}
}
if config.AdditionalQueries == nil {
// No additional queries set
return queries, nil
@@ -1004,12 +464,19 @@ func (svc *Service) GetDistributedQueries(ctx context.Context) (map[string]strin
// provided fleet.Host appropriately.
func (svc *Service) ingestDetailQuery(host *fleet.Host, name string, rows []map[string]string) error {
trimmedQuery := strings.TrimPrefix(name, hostDetailQueryPrefix)
config, err := svc.ds.AppConfig()
if err != nil {
return osqueryError{message: "ingest detail query: " + err.Error()}
}
detailQueries := osquery_utils.GetDetailQueries(config)
query, ok := detailQueries[trimmedQuery]
if !ok {
return osqueryError{message: "unknown detail query " + trimmedQuery}
}
err := query.IngestFunc(svc.logger, host, rows)
err = query.IngestFunc(svc.logger, host, rows)
if err != nil {
return osqueryError{
message: fmt.Sprintf("ingesting query %s: %s", name, err.Error()),
@@ -1025,7 +492,7 @@ func (svc *Service) ingestDetailQuery(host *fleet.Host, name string, rows []map[
// ingestLabelQuery records the results of label queries run by a host
func (svc *Service) ingestLabelQuery(host fleet.Host, query string, rows []map[string]string, results map[uint]bool) error {
trimmedQuery := strings.TrimPrefix(query, hostLabelQueryPrefix)
trimmedQueryNum, err := strconv.Atoi(emptyToZero(trimmedQuery))
trimmedQueryNum, err := strconv.Atoi(osquery_utils.EmptyToZero(trimmedQuery))
if err != nil {
return errors.Wrap(err, "converting query from string to int")
}
@@ -1040,7 +507,7 @@ func (svc *Service) ingestLabelQuery(host fleet.Host, query string, rows []map[s
func (svc *Service) ingestDistributedQuery(host fleet.Host, name string, rows []map[string]string, failed bool, errMsg string) error {
trimmedQuery := strings.TrimPrefix(name, hostDistributedQueryPrefix)
campaignID, err := strconv.Atoi(emptyToZero(trimmedQuery))
campaignID, err := strconv.Atoi(osquery_utils.EmptyToZero(trimmedQuery))
if err != nil {
return osqueryError{message: "unable to parse campaign ID: " + trimmedQuery}
}
+16 -261
View File
@@ -6,7 +6,6 @@ import (
"encoding/json"
"errors"
"fmt"
"sort"
"strconv"
"strings"
"sync"
@@ -25,14 +24,14 @@ import (
"github.com/fleetdm/fleet/v4/server/mock"
"github.com/fleetdm/fleet/v4/server/ptr"
"github.com/fleetdm/fleet/v4/server/pubsub"
"github.com/fleetdm/fleet/v4/server/service/osquery_utils"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// 3 detail queries are currently feature flagged off by default.
var expectedDetailQueries = len(detailQueries) - 3
var expectedDetailQueries = len(osquery_utils.GetDetailQueries(&fleet.AppConfig{EnableHostUsers: true}))
func TestEnrollAgent(t *testing.T) {
ds := new(mock.Store)
@@ -50,6 +49,9 @@ func TestEnrollAgent(t *testing.T) {
OsqueryHostID: osqueryHostId, NodeKey: nodeKey,
}, nil
}
ds.AppConfigFunc = func() (*fleet.AppConfig, error) {
return &fleet.AppConfig{EnableHostUsers: true}, nil
}
svc := newTestService(ds, nil, nil)
@@ -91,6 +93,9 @@ func TestEnrollAgentDetails(t *testing.T) {
gotHost = host
return nil
}
ds.AppConfigFunc = func() (*fleet.AppConfig, error) {
return &fleet.AppConfig{EnableHostUsers: true}, nil
}
svc := newTestService(ds, nil, nil)
@@ -246,7 +251,7 @@ func TestHostDetailQueries(t *testing.T) {
ds := new(mock.Store)
additional := json.RawMessage(`{"foobar": "select foo", "bim": "bam"}`)
ds.AppConfigFunc = func() (*fleet.AppConfig, error) {
return &fleet.AppConfig{AdditionalQueries: &additional}, nil
return &fleet.AppConfig{AdditionalQueries: &additional, EnableHostUsers: true}, nil
}
mockClock := clock.NewMockClock()
@@ -324,7 +329,7 @@ func TestLabelQueries(t *testing.T) {
return nil
}
ds.AppConfigFunc = func() (*fleet.AppConfig, error) {
return &fleet.AppConfig{}, nil
return &fleet.AppConfig{EnableHostUsers: true}, nil
}
lq.On("QueriesForHost", uint(0)).Return(map[string]string{}, nil)
@@ -524,7 +529,7 @@ func TestDetailQueriesWithEmptyStrings(t *testing.T) {
ctx := hostctx.NewContext(context.Background(), host)
ds.AppConfigFunc = func() (*fleet.AppConfig, error) {
return &fleet.AppConfig{}, nil
return &fleet.AppConfig{EnableHostUsers: true}, nil
}
ds.LabelQueriesForHostFunc = func(*fleet.Host, time.Time) (map[string]string, error) {
return map[string]string{}, nil
@@ -701,7 +706,7 @@ func TestDetailQueries(t *testing.T) {
lq.On("QueriesForHost", host.ID).Return(map[string]string{}, nil)
ds.AppConfigFunc = func() (*fleet.AppConfig, error) {
return &fleet.AppConfig{}, nil
return &fleet.AppConfig{EnableHostUsers: true}, nil
}
ds.LabelQueriesForHostFunc = func(*fleet.Host, time.Time) (map[string]string, error) {
return map[string]string{}, nil
@@ -886,259 +891,6 @@ func TestDetailQueries(t *testing.T) {
assert.Zero(t, acc)
}
func TestDetailQueryNetworkInterfaces(t *testing.T) {
var initialHost fleet.Host
host := initialHost
ingest := detailQueries["network_interface"].IngestFunc
assert.NoError(t, ingest(log.NewNopLogger(), &host, nil))
assert.Equal(t, initialHost, host)
var rows []map[string]string
require.NoError(t, json.Unmarshal([]byte(`
[
{"address":"127.0.0.1","mac":"00:00:00:00:00:00"},
{"address":"::1","mac":"00:00:00:00:00:00"},
{"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"},
{"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"},
{"address":"192.168.1.3","mac":"f4:5d:79:93:58:5b"},
{"address":"fe80::241a:9aff:fe60:d80a%awdl0","mac":"27:1b:aa:60:e8:0a"},
{"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"}
]`),
&rows,
))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.Equal(t, "192.168.1.3", host.PrimaryIP)
assert.Equal(t, "f4:5d:79:93:58:5b", host.PrimaryMac)
// Only IPv6
require.NoError(t, json.Unmarshal([]byte(`
[
{"address":"127.0.0.1","mac":"00:00:00:00:00:00"},
{"address":"::1","mac":"00:00:00:00:00:00"},
{"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"},
{"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"},
{"address":"2604:3f08:1337:9411:cbe:814f:51a6:e4e3","mac":"27:1b:aa:60:e8:0a"},
{"address":"3333:3f08:1337:9411:cbe:814f:51a6:e4e3","mac":"bb:1b:aa:60:e8:bb"},
{"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"}
]`),
&rows,
))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.Equal(t, "2604:3f08:1337:9411:cbe:814f:51a6:e4e3", host.PrimaryIP)
assert.Equal(t, "27:1b:aa:60:e8:0a", host.PrimaryMac)
// IPv6 appears before IPv4 (v4 should be prioritized)
require.NoError(t, json.Unmarshal([]byte(`
[
{"address":"127.0.0.1","mac":"00:00:00:00:00:00"},
{"address":"::1","mac":"00:00:00:00:00:00"},
{"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"},
{"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"},
{"address":"2604:3f08:1337:9411:cbe:814f:51a6:e4e3","mac":"27:1b:aa:60:e8:0a"},
{"address":"205.111.43.79","mac":"ab:1b:aa:60:e8:0a"},
{"address":"205.111.44.80","mac":"bb:bb:aa:60:e8:0a"},
{"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"}
]`),
&rows,
))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.Equal(t, "205.111.43.79", host.PrimaryIP)
assert.Equal(t, "ab:1b:aa:60:e8:0a", host.PrimaryMac)
// Only link-local/loopback
require.NoError(t, json.Unmarshal([]byte(`
[
{"address":"127.0.0.1","mac":"00:00:00:00:00:00"},
{"address":"::1","mac":"00:00:00:00:00:00"},
{"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"},
{"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"},
{"address":"fe80::241a:9aff:fe60:d80a%awdl0","mac":"27:1b:aa:60:e8:0a"},
{"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"}
]`),
&rows,
))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.Equal(t, "127.0.0.1", host.PrimaryIP)
assert.Equal(t, "00:00:00:00:00:00", host.PrimaryMac)
}
func TestDetailQueryScheduledQueryStats(t *testing.T) {
host := fleet.Host{}
ingest := detailQueries["scheduled_query_stats"].IngestFunc
assert.NoError(t, ingest(log.NewNopLogger(), &host, nil))
assert.Len(t, host.PackStats, 0)
resJSON := `
[
{
"average_memory":"33",
"delimiter":"/",
"denylisted":"0",
"executions":"1",
"interval":"33",
"last_executed":"1620325191",
"name":"pack/pack-2/time",
"output_size":"",
"query":"SELECT * FROM time",
"system_time":"100",
"user_time":"60",
"wall_time":"180"
},
{
"average_memory":"8000",
"delimiter":"/",
"denylisted":"0",
"executions":"164",
"interval":"30",
"last_executed":"1620325191",
"name":"pack/test/osquery info",
"output_size":"1337",
"query":"SELECT * FROM osquery_info",
"system_time":"150",
"user_time":"180",
"wall_time":"0"
},
{
"average_memory":"50400",
"delimiter":"/",
"denylisted":"1",
"executions":"188",
"interval":"30",
"last_executed":"1620325203",
"name":"pack/test/processes?",
"output_size":"",
"query":"SELECT * FROM processes",
"system_time":"140",
"user_time":"190",
"wall_time":"1"
},
{
"average_memory":"0",
"delimiter":"/",
"denylisted":"0",
"executions":"1",
"interval":"3600",
"last_executed":"1620323381",
"name":"pack/test/processes?-1",
"output_size":"",
"query":"SELECT * FROM processes",
"system_time":"0",
"user_time":"0",
"wall_time":"0"
},
{
"average_memory":"0",
"delimiter":"/",
"denylisted":"0",
"executions":"105",
"interval":"47",
"last_executed":"1620325190",
"name":"pack/test/time",
"output_size":"",
"query":"SELECT * FROM time",
"system_time":"70",
"user_time":"50",
"wall_time":"1"
}
]
`
var rows []map[string]string
require.NoError(t, json.Unmarshal([]byte(resJSON), &rows))
assert.NoError(t, ingest(log.NewNopLogger(), &host, rows))
assert.Len(t, host.PackStats, 2)
sort.Slice(host.PackStats, func(i, j int) bool {
return host.PackStats[i].PackName < host.PackStats[j].PackName
})
assert.Equal(t, host.PackStats[0].PackName, "pack-2")
assert.ElementsMatch(t, host.PackStats[0].QueryStats,
[]fleet.ScheduledQueryStats{
{
ScheduledQueryName: "time",
PackName: "pack-2",
AverageMemory: 33,
Denylisted: false,
Executions: 1,
Interval: 33,
LastExecuted: time.Unix(1620325191, 0).UTC(),
OutputSize: 0,
SystemTime: 100,
UserTime: 60,
WallTime: 180,
},
},
)
assert.Equal(t, host.PackStats[1].PackName, "test")
assert.ElementsMatch(t, host.PackStats[1].QueryStats,
[]fleet.ScheduledQueryStats{
{
ScheduledQueryName: "osquery info",
PackName: "test",
AverageMemory: 8000,
Denylisted: false,
Executions: 164,
Interval: 30,
LastExecuted: time.Unix(1620325191, 0).UTC(),
OutputSize: 1337,
SystemTime: 150,
UserTime: 180,
WallTime: 0,
},
{
ScheduledQueryName: "processes?",
PackName: "test",
AverageMemory: 50400,
Denylisted: true,
Executions: 188,
Interval: 30,
LastExecuted: time.Unix(1620325203, 0).UTC(),
OutputSize: 0,
SystemTime: 140,
UserTime: 190,
WallTime: 1,
},
{
ScheduledQueryName: "processes?-1",
PackName: "test",
AverageMemory: 0,
Denylisted: false,
Executions: 1,
Interval: 3600,
LastExecuted: time.Unix(1620323381, 0).UTC(),
OutputSize: 0,
SystemTime: 0,
UserTime: 0,
WallTime: 0,
},
{
ScheduledQueryName: "time",
PackName: "test",
AverageMemory: 0,
Denylisted: false,
Executions: 105,
Interval: 47,
LastExecuted: time.Unix(1620325190, 0).UTC(),
OutputSize: 0,
SystemTime: 70,
UserTime: 50,
WallTime: 1,
},
},
)
assert.NoError(t, ingest(log.NewNopLogger(), &host, nil))
assert.Len(t, host.PackStats, 0)
}
func TestNewDistributedQueryCampaign(t *testing.T) {
ds := &mock.Store{
AppConfigStore: mock.AppConfigStore{
@@ -1233,7 +985,7 @@ func TestDistributedQueryResults(t *testing.T) {
return nil
}
ds.AppConfigFunc = func() (*fleet.AppConfig, error) {
return &fleet.AppConfig{}, nil
return &fleet.AppConfig{EnableHostUsers: true}, nil
}
host := &fleet.Host{ID: 1, Platform: "windows"}
@@ -1882,6 +1634,9 @@ func TestDistributedQueriesReloadsHostIfDetailsAreIn(t *testing.T) {
require.Equal(t, uint(42), id)
return &fleet.Host{ID: 42, Platform: "darwin", PrimaryIP: ip}, nil
}
ds.AppConfigFunc = func() (*fleet.AppConfig, error) {
return &fleet.AppConfig{EnableHostUsers: true}, nil
}
ctx := hostctx.NewContext(context.Background(), *host)
-21
View File
@@ -83,27 +83,6 @@ func newTestServiceWithClock(ds fleet.Datastore, rs fleet.QueryResultStore, lq f
return svc
}
func createTestAppConfig(t *testing.T, ds fleet.Datastore) *fleet.AppConfig {
config := &fleet.AppConfig{
OrgName: "Tyrell Corp",
OrgLogoURL: "https://tyrell.com/image.png",
ServerURL: "https://fleet.tyrell.com",
SMTPConfigured: true,
SMTPSenderAddress: "test@example.com",
SMTPServer: "smtp.tyrell.com",
SMTPPort: 587,
SMTPAuthenticationType: fleet.AuthTypeUserNamePassword,
SMTPUserName: "deckard",
SMTPPassword: "replicant",
SMTPVerifySSLCerts: true,
SMTPEnableTLS: true,
}
result, err := ds.NewAppConfig(config)
require.Nil(t, err)
require.NotNil(t, result)
return result
}
func createTestUsers(t *testing.T, ds fleet.Datastore) map[string]fleet.User {
users := make(map[string]fleet.User)
for _, u := range testUsers {