From 11dff7ca92ca092a039347ab578a1fc4e1b26d0e Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Wed, 11 Aug 2021 14:56:11 -0300 Subject: [PATCH] Issue 1588 allow disabling host users (#1611) * Allow users to disable host_users * Add missing files * Address review comments * Fix tests * Check additional queries for nil, not the whole hostsettings * Finally fix tests --- changes/issue-1588-allow-disabling-host-users | 1 + cmd/fleetctl/apply_test.go | 58 ++ cmd/fleetctl/get_test.go | 62 ++ go.sum | 1 + server/datastore/mysql/app_configs.go | 9 +- .../20210810095603_AddEnableHostUsers.go | 22 + server/fleet/app.go | 10 +- server/service/endpoint_appconfig.go | 22 +- server/service/integration_test.go | 4 +- server/service/osquery_utils/queries.go | 576 +++++++++++++++++ server/service/osquery_utils/queries_test.go | 303 +++++++++ server/service/osquery_utils/utils.go | 11 + server/service/service_appconfig.go | 5 +- server/service/service_osquery.go | 601 +----------------- server/service/service_osquery_test.go | 277 +------- server/service/testing_utils.go | 21 - 16 files changed, 1112 insertions(+), 871 deletions(-) create mode 100644 changes/issue-1588-allow-disabling-host-users create mode 100644 server/datastore/mysql/migrations/tables/20210810095603_AddEnableHostUsers.go create mode 100644 server/service/osquery_utils/queries.go create mode 100644 server/service/osquery_utils/queries_test.go create mode 100644 server/service/osquery_utils/utils.go diff --git a/changes/issue-1588-allow-disabling-host-users b/changes/issue-1588-allow-disabling-host-users new file mode 100644 index 0000000000..dda1ff3b63 --- /dev/null +++ b/changes/issue-1588-allow-disabling-host-users @@ -0,0 +1 @@ +* In some cases, host users are not useful. Users can disable this through the enable_host_users config. diff --git a/cmd/fleetctl/apply_test.go b/cmd/fleetctl/apply_test.go index 74ddd5c6f8..2426acbdc7 100644 --- a/cmd/fleetctl/apply_test.go +++ b/cmd/fleetctl/apply_test.go @@ -174,3 +174,61 @@ spec: assert.Equal(t, &newAgentOpts, teamsByName["team1"].AgentOptions) assert.Equal(t, []*fleet.EnrollSecret{{Secret: "AAA"}}, enrolledSecretsCalled[uint(42)]) } + +func writeTmpYml(t *testing.T, contents string) string { + tmpFile, err := ioutil.TempFile(t.TempDir(), "*.yml") + require.NoError(t, err) + _, err = tmpFile.WriteString(contents) + require.NoError(t, err) + return tmpFile.Name() +} + +func TestApplyAppConfig(t *testing.T) { + server, ds := runServerWithMockedDS(t) + defer server.Close() + + ds.ListUsersFunc = func(opt fleet.UserListOptions) ([]*fleet.User, error) { + return userRoleSpecList, nil + } + + ds.UserByEmailFunc = func(email string) (*fleet.User, error) { + if email == "admin1@example.com" { + return userRoleSpecList[0], nil + } + return userRoleSpecList[1], nil + } + + ds.AppConfigFunc = func() (*fleet.AppConfig, error) { + return &fleet.AppConfig{}, nil + } + + var savedAppConfig *fleet.AppConfig + ds.SaveAppConfigFunc = func(config *fleet.AppConfig) error { + savedAppConfig = config + return nil + } + + name := writeTmpYml(t, `--- +apiVersion: v1 +kind: config +spec: + host_settings: + enable_host_users: false +`) + + assert.Equal(t, "[+] applied fleet config\n", runAppForTest(t, []string{"apply", "-f", name})) + require.NotNil(t, savedAppConfig) + assert.False(t, savedAppConfig.EnableHostUsers) + + name = writeTmpYml(t, `--- +apiVersion: v1 +kind: config +spec: + host_settings: + enable_host_users: true +`) + + assert.Equal(t, "[+] applied fleet config\n", runAppForTest(t, []string{"apply", "-f", name})) + require.NotNil(t, savedAppConfig) + assert.True(t, savedAppConfig.EnableHostUsers) +} diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index 7b36e0a3cd..60590d8f42 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -239,3 +239,65 @@ spec: assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "hosts", "--yaml"})) assert.Equal(t, expectedJson, runAppForTest(t, []string{"get", "hosts", "--json"})) } + +func TestGetConfig(t *testing.T) { + server, ds := runServerWithMockedDS(t) + defer server.Close() + + ds.AppConfigFunc = func() (*fleet.AppConfig, error) { + return &fleet.AppConfig{ + EnableHostUsers: true, + VulnerabilityDatabasesPath: ptr.String("/some/path"), + }, nil + } + + expectedYaml := `--- +apiVersion: v1 +kind: config +spec: + agent_options: null + host_expiry_settings: + host_expiry_enabled: false + host_expiry_window: 0 + host_settings: + enable_host_users: true + org_info: + org_logo_url: "" + org_name: "" + server_settings: + enable_analytics: false + live_query_disabled: false + server_url: "" + smtp_settings: + authentication_method: authmethod_plain + authentication_type: authtype_username_password + configured: false + domain: "" + enable_smtp: false + enable_ssl_tls: false + enable_start_tls: false + password: '********' + port: 0 + sender_address: "" + server: "" + user_name: "" + verify_ssl_certs: false + sso_settings: + enable_sso: false + enable_sso_idp_login: false + entity_id: "" + idp_image_url: "" + idp_name: "" + issuer_uri: "" + metadata: "" + metadata_url: "" + vulnerability_settings: + databases_path: /some/path +` + expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"authtype_username_password","user_name":"","password":"********","enable_ssl_tls":false,"authentication_method":"authmethod_plain","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true},"agent_options":null,"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"}}} +` + + assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "config"})) + assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "config", "--yaml"})) + assert.Equal(t, expectedJson, runAppForTest(t, []string{"get", "config", "--json"})) +} diff --git a/go.sum b/go.sum index d874c7985c..eb34ade776 100644 --- a/go.sum +++ b/go.sum @@ -854,6 +854,7 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c h1:TWQ2UvXPkhPxI2KmApKBOCaV6yD2N4mlvqFQ/DlPtpQ= github.com/zwass/kit v0.0.0-20210625184505-ec5b5c5cce9c/go.mod h1:OYYulo9tUqRadRLwB0+LE914sa1ui2yL7OrcU3Q/1XY= diff --git a/server/datastore/mysql/app_configs.go b/server/datastore/mysql/app_configs.go index f0fc2c811e..b5055dc396 100644 --- a/server/datastore/mysql/app_configs.go +++ b/server/datastore/mysql/app_configs.go @@ -123,9 +123,10 @@ func (d *Datastore) SaveAppConfig(info *fleet.AppConfig) error { additional_queries, agent_options, enable_analytics, - vulnerability_databases_path + vulnerability_databases_path, + enable_host_users ) - VALUES( 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) + VALUES( 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) ON DUPLICATE KEY UPDATE org_name = VALUES(org_name), org_logo_url = VALUES(org_logo_url), @@ -158,7 +159,8 @@ func (d *Datastore) SaveAppConfig(info *fleet.AppConfig) error { additional_queries = VALUES(additional_queries), agent_options = VALUES(agent_options), enable_analytics = VALUES(enable_analytics), - vulnerability_databases_path = VALUES(vulnerability_databases_path) + vulnerability_databases_path = VALUES(vulnerability_databases_path), + enable_host_users = VALUES(enable_host_users) ` _, err = tx.Exec(insertStatement, @@ -194,6 +196,7 @@ func (d *Datastore) SaveAppConfig(info *fleet.AppConfig) error { info.AgentOptions, info.EnableAnalytics, info.VulnerabilityDatabasesPath, + info.EnableHostUsers, ) if err != nil { return err diff --git a/server/datastore/mysql/migrations/tables/20210810095603_AddEnableHostUsers.go b/server/datastore/mysql/migrations/tables/20210810095603_AddEnableHostUsers.go new file mode 100644 index 0000000000..06d377a22a --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20210810095603_AddEnableHostUsers.go @@ -0,0 +1,22 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20210810095603, Down_20210810095603) +} + +func Up_20210810095603(tx *sql.Tx) error { + if _, err := tx.Exec(`ALTER TABLE app_configs ADD COLUMN enable_host_users BOOL DEFAULT TRUE`); err != nil { + return errors.Wrap(err, "add column enable_host_users") + } + return nil +} + +func Down_20210810095603(tx *sql.Tx) error { + return nil +} diff --git a/server/fleet/app.go b/server/fleet/app.go index e06559142b..b1d7d4f083 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -186,6 +186,9 @@ type AppConfig struct { // VulnerabilityDatabases path VulnerabilityDatabasesPath *string `db:"vulnerability_databases_path"` + + // EnableHostUsers indicates whether the users of each host will be queried and stored + EnableHostUsers bool `db:"enable_host_users" json:"enable_host_users"` } func (c AppConfig) AuthzType() string { @@ -309,7 +312,8 @@ type HostExpirySettings struct { } type HostSettings struct { - AdditionalQueries *json.RawMessage `json:"additional_queries"` + EnableHostUsers *bool `json:"enable_host_users"` + AdditionalQueries *json.RawMessage `json:"additional_queries,omitempty"` } type OrderDirection int @@ -398,7 +402,7 @@ type Logging struct { } type LoggingPlugin struct { - Plugin string `json:"plugin"` + Plugin string `json:"plugin"` Config interface{} `json:"config"` } @@ -430,5 +434,3 @@ type LambdaConfig struct { StatusFunction string `json:"status_function"` ResultFunction string `json:"result_function"` } - - diff --git a/server/service/endpoint_appconfig.go b/server/service/endpoint_appconfig.go index b038ada26d..c3b4079bf9 100644 --- a/server/service/endpoint_appconfig.go +++ b/server/service/endpoint_appconfig.go @@ -27,8 +27,8 @@ type appConfigResponse struct { VulnerabilitySettings *fleet.VulnerabilitySettingsPayload `json:"vulnerability_settings"` // Logging is loaded on the fly rather than from the database. - Logging *fleet.Logging `json:"logging,omitempty"` - Err error `json:"error,omitempty"` + Logging *fleet.Logging `json:"logging,omitempty"` + Err error `json:"error,omitempty"` } func (r appConfigResponse) error() error { return r.Err } @@ -85,11 +85,9 @@ func makeGetAppConfigEndpoint(svc fleet.Service) endpoint.Endpoint { } } hostSettings := &fleet.HostSettings{ + EnableHostUsers: &config.EnableHostUsers, AdditionalQueries: config.AdditionalQueries, } - if config.AdditionalQueries == nil { - hostSettings = nil - } response := appConfigResponse{ OrgInfo: &fleet.OrgInfo{ OrgName: &config.OrgName, @@ -100,13 +98,13 @@ func makeGetAppConfigEndpoint(svc fleet.Service) endpoint.Endpoint { LiveQueryDisabled: &config.LiveQueryDisabled, EnableAnalytics: &config.EnableAnalytics, }, - SMTPSettings: smtpSettings, - SSOSettings: ssoSettings, - HostExpirySettings: hostExpirySettings, - HostSettings: hostSettings, - License: license, - AgentOptions: agentOptions, - Logging: loggingConfig, + SMTPSettings: smtpSettings, + SSOSettings: ssoSettings, + HostExpirySettings: hostExpirySettings, + HostSettings: hostSettings, + License: license, + AgentOptions: agentOptions, + Logging: loggingConfig, VulnerabilitySettings: vulnerabilitySettings, } return response, nil diff --git a/server/service/integration_test.go b/server/service/integration_test.go index dd1bcddd65..8a187e65d0 100644 --- a/server/service/integration_test.go +++ b/server/service/integration_test.go @@ -253,6 +253,7 @@ func TestAppConfigAdditionalQueriesCanBeRemoved(t *testing.T) { host_settings: additional_queries: time: SELECT * FROM time + enable_host_users: true `) applyConfig(t, spec, server, token) @@ -261,11 +262,12 @@ func TestAppConfigAdditionalQueriesCanBeRemoved(t *testing.T) { host_expiry_enabled: false host_expiry_window: 0 host_settings: + enable_host_users: true `) applyConfig(t, spec, server, token) config := getConfig(t, server, token) - assert.Nil(t, config.HostSettings) + assert.Nil(t, config.HostSettings.AdditionalQueries) } func TestAppConfigHasLogging(t *testing.T) { diff --git a/server/service/osquery_utils/queries.go b/server/service/osquery_utils/queries.go new file mode 100644 index 0000000000..be2270ad12 --- /dev/null +++ b/server/service/osquery_utils/queries.go @@ -0,0 +1,576 @@ +package osquery_utils + +import ( + "fmt" + "net" + "os" + "strconv" + "strings" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/pkg/errors" + "github.com/spf13/cast" +) + +type DetailQuery struct { + Query string + // Platforms is a list of platforms to run the query on. If this value is + // empty, run on all platforms. + Platforms []string + IngestFunc func(logger log.Logger, host *fleet.Host, rows []map[string]string) error +} + +// RunsForPlatform determines whether this detail query should run on the given platform +func (q *DetailQuery) RunsForPlatform(platform string) bool { + if len(q.Platforms) == 0 { + return true + } + for _, p := range q.Platforms { + if p == platform { + return true + } + } + return false +} + +// detailQueries defines the detail queries that should be run on the host, as +// well as how the results of those queries should be ingested into the +// fleet.Host data model. This map should not be modified at runtime. +var detailQueries = map[string]DetailQuery{ + "network_interface": { + Query: `select address, mac + from interface_details id join interface_addresses ia + on ia.interface = id.interface where length(mac) > 0 + order by (ibytes + obytes) desc`, + IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) (err error) { + if len(rows) == 0 { + logger.Log("component", "service", "method", "IngestFunc", "err", + "detail_query_network_interface expected 1 or more results") + return nil + } + + // Rows are ordered by traffic, so we will get the most active + // interface by iterating in order + var firstIPv4, firstIPv6 map[string]string + for _, row := range rows { + ip := net.ParseIP(row["address"]) + if ip == nil { + continue + } + + // Skip link-local and loopback interfaces + if ip.IsLinkLocalUnicast() || ip.IsLoopback() { + continue + } + + if strings.Contains(row["address"], ":") { + //IPv6 + if firstIPv6 == nil { + firstIPv6 = row + } + } else { + // IPv4 + if firstIPv4 == nil { + firstIPv4 = row + } + } + } + + var selected map[string]string + switch { + // Prefer IPv4 + case firstIPv4 != nil: + selected = firstIPv4 + // Otherwise IPv6 + case firstIPv6 != nil: + selected = firstIPv6 + // If only link-local and loopback found, still use the first + // interface so that we don't get an empty value. + default: + selected = rows[0] + } + + host.PrimaryIP = selected["address"] + host.PrimaryMac = selected["mac"] + return nil + }, + }, + "os_version": { + Query: "select * from os_version limit 1", + IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { + if len(rows) != 1 { + logger.Log("component", "service", "method", "IngestFunc", "err", + fmt.Sprintf("detail_query_os_version expected single result got %d", len(rows))) + return nil + } + + host.OSVersion = fmt.Sprintf( + "%s %s.%s.%s", + rows[0]["name"], + rows[0]["major"], + rows[0]["minor"], + rows[0]["patch"], + ) + host.OSVersion = strings.Trim(host.OSVersion, ".") + + if build, ok := rows[0]["build"]; ok { + host.Build = build + } + + host.Platform = rows[0]["platform"] + host.PlatformLike = rows[0]["platform_like"] + host.CodeName = rows[0]["code_name"] + + // On centos6 there is an osquery bug that leaves + // platform empty. Here we workaround. + if host.Platform == "" && + strings.Contains(strings.ToLower(rows[0]["name"]), "centos") { + host.Platform = "centos" + } + + return nil + }, + }, + "osquery_flags": { + // Collect the interval info (used for online status + // calculation) from the osquery flags. We typically control + // distributed_interval (but it's not required), and typically + // do not control config_tls_refresh. + Query: `select name, value from osquery_flags where name in ("distributed_interval", "config_tls_refresh", "config_refresh", "logger_tls_period")`, + IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { + var configTLSRefresh, configRefresh uint + var configRefreshSeen, configTLSRefreshSeen bool + for _, row := range rows { + switch row["name"] { + + case "distributed_interval": + interval, err := strconv.Atoi(EmptyToZero(row["value"])) + if err != nil { + return errors.Wrap(err, "parsing distributed_interval") + } + host.DistributedInterval = uint(interval) + + case "config_tls_refresh": + // Prior to osquery 2.4.6, the flag was + // called `config_tls_refresh`. + interval, err := strconv.Atoi(EmptyToZero(row["value"])) + if err != nil { + return errors.Wrap(err, "parsing config_tls_refresh") + } + configTLSRefresh = uint(interval) + configTLSRefreshSeen = true + + case "config_refresh": + // After 2.4.6 `config_tls_refresh` was + // aliased to `config_refresh`. + interval, err := strconv.Atoi(EmptyToZero(row["value"])) + if err != nil { + return errors.Wrap(err, "parsing config_refresh") + } + configRefresh = uint(interval) + configRefreshSeen = true + + case "logger_tls_period": + interval, err := strconv.Atoi(EmptyToZero(row["value"])) + if err != nil { + return errors.Wrap(err, "parsing logger_tls_period") + } + host.LoggerTLSPeriod = uint(interval) + } + } + + // Since the `config_refresh` flag existed prior to + // 2.4.6 and had a different meaning, we prefer + // `config_tls_refresh` if it was set, and use + // `config_refresh` as a fallback. + if configTLSRefreshSeen { + host.ConfigTLSRefresh = configTLSRefresh + } else if configRefreshSeen { + host.ConfigTLSRefresh = configRefresh + } + + return nil + }, + }, + "osquery_info": { + Query: "select * from osquery_info limit 1", + IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { + if len(rows) != 1 { + logger.Log("component", "service", "method", "IngestFunc", "err", + fmt.Sprintf("detail_query_osquery_info expected single result got %d", len(rows))) + return nil + } + + host.OsqueryVersion = rows[0]["version"] + + return nil + }, + }, + "system_info": { + Query: "select * from system_info limit 1", + IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { + if len(rows) != 1 { + logger.Log("component", "service", "method", "IngestFunc", "err", + fmt.Sprintf("detail_query_system_info expected single result got %d", len(rows))) + return nil + } + + var err error + host.Memory, err = strconv.ParseInt(EmptyToZero(rows[0]["physical_memory"]), 10, 64) + if err != nil { + return err + } + host.Hostname = rows[0]["hostname"] + host.UUID = rows[0]["uuid"] + host.CPUType = rows[0]["cpu_type"] + host.CPUSubtype = rows[0]["cpu_subtype"] + host.CPUBrand = rows[0]["cpu_brand"] + host.CPUPhysicalCores, err = strconv.Atoi(EmptyToZero(rows[0]["cpu_physical_cores"])) + if err != nil { + return err + } + host.CPULogicalCores, err = strconv.Atoi(EmptyToZero(rows[0]["cpu_logical_cores"])) + if err != nil { + return err + } + host.HardwareVendor = rows[0]["hardware_vendor"] + host.HardwareModel = rows[0]["hardware_model"] + host.HardwareVersion = rows[0]["hardware_version"] + host.HardwareSerial = rows[0]["hardware_serial"] + host.ComputerName = rows[0]["computer_name"] + return nil + }, + }, + "uptime": { + Query: "select * from uptime limit 1", + IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { + if len(rows) != 1 { + logger.Log("component", "service", "method", "IngestFunc", "err", + fmt.Sprintf("detail_query_uptime expected single result got %d", len(rows))) + return nil + } + + uptimeSeconds, err := strconv.Atoi(EmptyToZero(rows[0]["total_seconds"])) + if err != nil { + return err + } + host.Uptime = time.Duration(uptimeSeconds) * time.Second + + return nil + }, + }, + "scheduled_query_stats": { + Query: ` + SELECT *, + (SELECT value from osquery_flags where name = 'pack_delimiter') AS delimiter + FROM osquery_schedule +`, + IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { + packs := map[string][]fleet.ScheduledQueryStats{} + + for _, row := range rows { + providedName := row["name"] + if providedName == "" { + level.Debug(logger).Log( + "msg", "host reported scheduled query with empty name", + "host", host.Hostname, + ) + continue + } + delimiter := row["delimiter"] + if delimiter == "" { + level.Debug(logger).Log( + "msg", "host reported scheduled query with empty delimiter", + "host", host.Hostname, + ) + continue + } + + // Split with a limit of 2 in case query name includes the + // delimiter. Not much we can do if pack name includes the + // delimiter. + trimmedName := strings.TrimPrefix(providedName, "pack"+delimiter) + parts := strings.SplitN(trimmedName, delimiter, 2) + if len(parts) != 2 { + level.Debug(logger).Log( + "msg", "could not split pack and query names", + "host", host.Hostname, + "name", providedName, + "delimiter", delimiter, + ) + continue + } + packName, scheduledName := parts[0], parts[1] + + stats := fleet.ScheduledQueryStats{ + ScheduledQueryName: scheduledName, + PackName: packName, + AverageMemory: cast.ToInt(row["average_memory"]), + Denylisted: cast.ToBool(row["denylisted"]), + Executions: cast.ToInt(row["executions"]), + Interval: cast.ToInt(row["interval"]), + // Cast to int first to allow cast.ToTime to interpret the unix timestamp. + LastExecuted: time.Unix(cast.ToInt64(row["last_executed"]), 0).UTC(), + OutputSize: cast.ToInt(row["output_size"]), + SystemTime: cast.ToInt(row["system_time"]), + UserTime: cast.ToInt(row["user_time"]), + WallTime: cast.ToInt(row["wall_time"]), + } + packs[packName] = append(packs[packName], stats) + } + + host.PackStats = []fleet.PackStats{} + for packName, stats := range packs { + host.PackStats = append( + host.PackStats, + fleet.PackStats{ + PackName: packName, + QueryStats: stats, + }, + ) + } + + return nil + }, + }, +} + +var softwareMacOS = DetailQuery{ + Query: ` +SELECT + name AS name, + bundle_short_version AS version, + 'Application (macOS)' AS type, + 'apps' AS source +FROM apps +UNION +SELECT + name AS name, + version AS version, + 'Package (Python)' AS type, + 'python_packages' AS source +FROM python_packages +UNION +SELECT + name AS name, + version AS version, + 'Browser plugin (Chrome)' AS type, + 'chrome_extensions' AS source +FROM chrome_extensions +UNION +SELECT + name AS name, + version AS version, + 'Browser plugin (Firefox)' AS type, + 'firefox_addons' AS source +FROM firefox_addons +UNION +SELECT + name As name, + version AS version, + 'Browser plugin (Safari)' AS type, + 'safari_extensions' AS source +FROM safari_extensions +UNION +SELECT + name AS name, + version AS version, + 'Package (Homebrew)' AS type, + 'homebrew_packages' AS source +FROM homebrew_packages; +`, + Platforms: []string{"darwin"}, + IngestFunc: ingestSoftware, +} + +var softwareLinux = DetailQuery{ + Query: ` +SELECT + name AS name, + version AS version, + 'Package (deb)' AS type, + 'deb_packages' AS source +FROM deb_packages +UNION +SELECT + package AS name, + version AS version, + 'Package (Portage)' AS type, + 'portage_packages' AS source +FROM portage_packages +UNION +SELECT + name AS name, + version AS version, + 'Package (RPM)' AS type, + 'rpm_packages' AS source +FROM rpm_packages +UNION +SELECT + name AS name, + version AS version, + 'Package (NPM)' AS type, + 'npm_packages' AS source +FROM npm_packages +UNION +SELECT + name AS name, + version AS version, + 'Package (Atom)' AS type, + 'atom_packages' AS source +FROM atom_packages +UNION +SELECT + name AS name, + version AS version, + 'Package (Python)' AS type, + 'python_packages' AS source +FROM python_packages; +`, + Platforms: []string{"linux", "rhel", "ubuntu", "centos"}, + IngestFunc: ingestSoftware, +} + +var softwareWindows = DetailQuery{ + Query: ` +SELECT + name AS name, + version AS version, + 'Program (Windows)' AS type, + 'programs' AS source +FROM programs +UNION +SELECT + name AS name, + version AS version, + 'Package (Python)' AS type, + 'python_packages' AS source +FROM python_packages +UNION +SELECT + name AS name, + version AS version, + 'Browser plugin (IE)' AS type, + 'ie_extensions' AS source +FROM ie_extensions +UNION +SELECT + name AS name, + version AS version, + 'Browser plugin (Chrome)' AS type, + 'chrome_extensions' AS source +FROM chrome_extensions +UNION +SELECT + name AS name, + version AS version, + 'Browser plugin (Firefox)' AS type, + 'firefox_addons' AS source +FROM firefox_addons +UNION +SELECT + name AS name, + version AS version, + 'Package (Chocolatey)' AS type, + 'chocolatey_packages' AS source +FROM chocolatey_packages +UNION +SELECT + name AS name, + version AS version, + 'Package (Atom)' AS type, + 'atom_packages' AS source +FROM atom_packages +UNION +SELECT + name AS name, + version AS version, + 'Package (Python)' AS type, + 'python_packages' AS source +FROM python_packages; +`, + Platforms: []string{"windows"}, + IngestFunc: ingestSoftware, +} + +var usersQuery = DetailQuery{ + Query: `SELECT uid, username, type, groupname FROM users u JOIN groups g ON g.gid=u.gid;`, + IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { + var users []fleet.HostUser + for _, row := range rows { + uid, err := strconv.Atoi(row["uid"]) + if err != nil { + return errors.Wrapf(err, "converting uid %s to int", row["uid"]) + } + username := row["username"] + type_ := row["type"] + groupname := row["groupname"] + u := fleet.HostUser{ + Uid: uint(uid), + Username: username, + Type: type_, + GroupName: groupname, + } + users = append(users, u) + } + host.Users = users + + return nil + }, +} + +func ingestSoftware(logger log.Logger, host *fleet.Host, rows []map[string]string) error { + software := fleet.HostSoftware{Modified: true} + + for _, row := range rows { + name := row["name"] + version := row["version"] + source := row["source"] + if name == "" { + level.Debug(logger).Log( + "msg", "host reported software with empty name", + "host", host.Hostname, + "version", version, + "source", source, + ) + continue + } + if source == "" { + level.Debug(logger).Log( + "msg", "host reported software with empty name", + "host", host.Hostname, + "version", version, + "name", name, + ) + continue + } + s := fleet.Software{Name: name, Version: version, Source: source} + software.Software = append(software.Software, s) + } + + host.HostSoftware = software + + return nil +} + +func GetDetailQueries(ac *fleet.AppConfig) map[string]DetailQuery { + generatedMap := make(map[string]DetailQuery) + for key, query := range detailQueries { + generatedMap[key] = query + } + + if os.Getenv("FLEET_BETA_SOFTWARE_INVENTORY") != "" { + generatedMap["software_macos"] = softwareMacOS + generatedMap["software_linux"] = softwareLinux + generatedMap["software_windows"] = softwareWindows + } + + if ac != nil && ac.EnableHostUsers { + generatedMap["users"] = usersQuery + } + + return generatedMap +} diff --git a/server/service/osquery_utils/queries_test.go b/server/service/osquery_utils/queries_test.go new file mode 100644 index 0000000000..4036ba1793 --- /dev/null +++ b/server/service/osquery_utils/queries_test.go @@ -0,0 +1,303 @@ +package osquery_utils + +import ( + "encoding/json" + "os" + "sort" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/go-kit/kit/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDetailQueryNetworkInterfaces(t *testing.T) { + var initialHost fleet.Host + host := initialHost + + ingest := GetDetailQueries(nil)["network_interface"].IngestFunc + + assert.NoError(t, ingest(log.NewNopLogger(), &host, nil)) + assert.Equal(t, initialHost, host) + + var rows []map[string]string + require.NoError(t, json.Unmarshal([]byte(` +[ + {"address":"127.0.0.1","mac":"00:00:00:00:00:00"}, + {"address":"::1","mac":"00:00:00:00:00:00"}, + {"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"}, + {"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"}, + {"address":"192.168.1.3","mac":"f4:5d:79:93:58:5b"}, + {"address":"fe80::241a:9aff:fe60:d80a%awdl0","mac":"27:1b:aa:60:e8:0a"}, + {"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"} +]`), + &rows, + )) + + assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) + assert.Equal(t, "192.168.1.3", host.PrimaryIP) + assert.Equal(t, "f4:5d:79:93:58:5b", host.PrimaryMac) + + // Only IPv6 + require.NoError(t, json.Unmarshal([]byte(` +[ + {"address":"127.0.0.1","mac":"00:00:00:00:00:00"}, + {"address":"::1","mac":"00:00:00:00:00:00"}, + {"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"}, + {"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"}, + {"address":"2604:3f08:1337:9411:cbe:814f:51a6:e4e3","mac":"27:1b:aa:60:e8:0a"}, + {"address":"3333:3f08:1337:9411:cbe:814f:51a6:e4e3","mac":"bb:1b:aa:60:e8:bb"}, + {"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"} +]`), + &rows, + )) + + assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) + assert.Equal(t, "2604:3f08:1337:9411:cbe:814f:51a6:e4e3", host.PrimaryIP) + assert.Equal(t, "27:1b:aa:60:e8:0a", host.PrimaryMac) + + // IPv6 appears before IPv4 (v4 should be prioritized) + require.NoError(t, json.Unmarshal([]byte(` +[ + {"address":"127.0.0.1","mac":"00:00:00:00:00:00"}, + {"address":"::1","mac":"00:00:00:00:00:00"}, + {"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"}, + {"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"}, + {"address":"2604:3f08:1337:9411:cbe:814f:51a6:e4e3","mac":"27:1b:aa:60:e8:0a"}, + {"address":"205.111.43.79","mac":"ab:1b:aa:60:e8:0a"}, + {"address":"205.111.44.80","mac":"bb:bb:aa:60:e8:0a"}, + {"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"} +]`), + &rows, + )) + + assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) + assert.Equal(t, "205.111.43.79", host.PrimaryIP) + assert.Equal(t, "ab:1b:aa:60:e8:0a", host.PrimaryMac) + + // Only link-local/loopback + require.NoError(t, json.Unmarshal([]byte(` +[ + {"address":"127.0.0.1","mac":"00:00:00:00:00:00"}, + {"address":"::1","mac":"00:00:00:00:00:00"}, + {"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"}, + {"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"}, + {"address":"fe80::241a:9aff:fe60:d80a%awdl0","mac":"27:1b:aa:60:e8:0a"}, + {"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"} +]`), + &rows, + )) + + assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) + assert.Equal(t, "127.0.0.1", host.PrimaryIP) + assert.Equal(t, "00:00:00:00:00:00", host.PrimaryMac) +} + +func TestDetailQueryScheduledQueryStats(t *testing.T) { + host := fleet.Host{} + + ingest := GetDetailQueries(nil)["scheduled_query_stats"].IngestFunc + + assert.NoError(t, ingest(log.NewNopLogger(), &host, nil)) + assert.Len(t, host.PackStats, 0) + + resJSON := ` +[ + { + "average_memory":"33", + "delimiter":"/", + "denylisted":"0", + "executions":"1", + "interval":"33", + "last_executed":"1620325191", + "name":"pack/pack-2/time", + "output_size":"", + "query":"SELECT * FROM time", + "system_time":"100", + "user_time":"60", + "wall_time":"180" + }, + { + "average_memory":"8000", + "delimiter":"/", + "denylisted":"0", + "executions":"164", + "interval":"30", + "last_executed":"1620325191", + "name":"pack/test/osquery info", + "output_size":"1337", + "query":"SELECT * FROM osquery_info", + "system_time":"150", + "user_time":"180", + "wall_time":"0" + }, + { + "average_memory":"50400", + "delimiter":"/", + "denylisted":"1", + "executions":"188", + "interval":"30", + "last_executed":"1620325203", + "name":"pack/test/processes?", + "output_size":"", + "query":"SELECT * FROM processes", + "system_time":"140", + "user_time":"190", + "wall_time":"1" + }, + { + "average_memory":"0", + "delimiter":"/", + "denylisted":"0", + "executions":"1", + "interval":"3600", + "last_executed":"1620323381", + "name":"pack/test/processes?-1", + "output_size":"", + "query":"SELECT * FROM processes", + "system_time":"0", + "user_time":"0", + "wall_time":"0" + }, + { + "average_memory":"0", + "delimiter":"/", + "denylisted":"0", + "executions":"105", + "interval":"47", + "last_executed":"1620325190", + "name":"pack/test/time", + "output_size":"", + "query":"SELECT * FROM time", + "system_time":"70", + "user_time":"50", + "wall_time":"1" + } +] +` + + var rows []map[string]string + require.NoError(t, json.Unmarshal([]byte(resJSON), &rows)) + + assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) + assert.Len(t, host.PackStats, 2) + sort.Slice(host.PackStats, func(i, j int) bool { + return host.PackStats[i].PackName < host.PackStats[j].PackName + }) + assert.Equal(t, host.PackStats[0].PackName, "pack-2") + assert.ElementsMatch(t, host.PackStats[0].QueryStats, + []fleet.ScheduledQueryStats{ + { + ScheduledQueryName: "time", + PackName: "pack-2", + AverageMemory: 33, + Denylisted: false, + Executions: 1, + Interval: 33, + LastExecuted: time.Unix(1620325191, 0).UTC(), + OutputSize: 0, + SystemTime: 100, + UserTime: 60, + WallTime: 180, + }, + }, + ) + assert.Equal(t, host.PackStats[1].PackName, "test") + assert.ElementsMatch(t, host.PackStats[1].QueryStats, + []fleet.ScheduledQueryStats{ + { + ScheduledQueryName: "osquery info", + PackName: "test", + AverageMemory: 8000, + Denylisted: false, + Executions: 164, + Interval: 30, + LastExecuted: time.Unix(1620325191, 0).UTC(), + OutputSize: 1337, + SystemTime: 150, + UserTime: 180, + WallTime: 0, + }, + { + ScheduledQueryName: "processes?", + PackName: "test", + AverageMemory: 50400, + Denylisted: true, + Executions: 188, + Interval: 30, + LastExecuted: time.Unix(1620325203, 0).UTC(), + OutputSize: 0, + SystemTime: 140, + UserTime: 190, + WallTime: 1, + }, + { + ScheduledQueryName: "processes?-1", + PackName: "test", + AverageMemory: 0, + Denylisted: false, + Executions: 1, + Interval: 3600, + LastExecuted: time.Unix(1620323381, 0).UTC(), + OutputSize: 0, + SystemTime: 0, + UserTime: 0, + WallTime: 0, + }, + { + ScheduledQueryName: "time", + PackName: "test", + AverageMemory: 0, + Denylisted: false, + Executions: 105, + Interval: 47, + LastExecuted: time.Unix(1620325190, 0).UTC(), + OutputSize: 0, + SystemTime: 70, + UserTime: 50, + WallTime: 1, + }, + }, + ) + + assert.NoError(t, ingest(log.NewNopLogger(), &host, nil)) + assert.Len(t, host.PackStats, 0) +} + +func sortedKeysCompare(t *testing.T, m map[string]DetailQuery, expectedKeys []string) { + var keys []string + for key := range m { + keys = append(keys, key) + } + assert.ElementsMatch(t, keys, expectedKeys) +} + +func TestGetDetailQueries(t *testing.T) { + queriesNoConfig := GetDetailQueries(nil) + require.Len(t, queriesNoConfig, 7) + baseQueries := []string{ + "network_interface", + "os_version", + "osquery_flags", + "osquery_info", + "scheduled_query_stats", + "system_info", + "uptime", + } + sortedKeysCompare(t, queriesNoConfig, baseQueries) + + queriesWithUsers := GetDetailQueries(&fleet.AppConfig{EnableHostUsers: true}) + require.Len(t, queriesWithUsers, 8) + sortedKeysCompare(t, queriesWithUsers, append(baseQueries, "users")) + + require.NoError(t, os.Setenv("FLEET_BETA_SOFTWARE_INVENTORY", "1")) + + queriesWithUsersAndSoftware := GetDetailQueries(&fleet.AppConfig{EnableHostUsers: true}) + require.Len(t, queriesWithUsersAndSoftware, 11) + sortedKeysCompare(t, queriesWithUsersAndSoftware, + append(baseQueries, "users", "software_macos", "software_linux", "software_windows")) + + require.NoError(t, os.Setenv("FLEET_BETA_SOFTWARE_INVENTORY", "")) +} diff --git a/server/service/osquery_utils/utils.go b/server/service/osquery_utils/utils.go new file mode 100644 index 0000000000..b22a152baf --- /dev/null +++ b/server/service/osquery_utils/utils.go @@ -0,0 +1,11 @@ +package osquery_utils + +// EmptyToZero Sometimes osquery gives us empty string where we expect an integer. +// We change the to "0" so it can be handled by the appropriate string to +// integer conversion function, as these will err on "" +func EmptyToZero(val string) string { + if val == "" { + return "0" + } + return val +} diff --git a/server/service/service_appconfig.go b/server/service/service_appconfig.go index 1e434ce913..3578f227f9 100644 --- a/server/service/service_appconfig.go +++ b/server/service/service_appconfig.go @@ -190,8 +190,9 @@ func appConfigFromAppConfigPayload(p fleet.AppConfigPayload, config fleet.AppCon } if settings := p.HostSettings; settings != nil { - if settings.AdditionalQueries != nil { - config.AdditionalQueries = settings.AdditionalQueries + config.AdditionalQueries = settings.AdditionalQueries + if settings.EnableHostUsers != nil { + config.EnableHostUsers = *settings.EnableHostUsers } } else { config.AdditionalQueries = nil diff --git a/server/service/service_osquery.go b/server/service/service_osquery.go index f35829daaa..2b9611d59d 100644 --- a/server/service/service_osquery.go +++ b/server/service/service_osquery.go @@ -4,14 +4,13 @@ import ( "context" "encoding/json" "fmt" - "net" - "os" "strconv" "strings" "time" "github.com/fleetdm/fleet/v4/server" "github.com/fleetdm/fleet/v4/server/contexts/logging" + "github.com/fleetdm/fleet/v4/server/service/osquery_utils" kithttp "github.com/go-kit/kit/transport/http" "github.com/fleetdm/fleet/v4/server/fleet" @@ -37,16 +36,6 @@ func (e osqueryError) NodeInvalid() bool { return e.nodeInvalid } -// Sometimes osquery gives us empty string where we expect an integer. -// We change the to "0" so it can be handled by the appropriate string to -// integer conversion function, as these will err on "" -func emptyToZero(val string) string { - if val == "" { - return "0" - } - return val -} - func (svc Service) AuthenticateHost(ctx context.Context, nodeKey string) (*fleet.Host, error) { // skipauth: Authorization is currently for user endpoints only. svc.authz.SkipAuthorization(ctx) @@ -116,18 +105,32 @@ func (svc Service) EnrollAgent(ctx context.Context, enrollSecret, hostIdentifier return "", osqueryError{message: "save enroll failed: " + err.Error(), nodeInvalid: true} } + appConfig, err := svc.ds.AppConfig() + if err != nil { + return "", osqueryError{message: "save enroll failed: " + err.Error(), nodeInvalid: true} + } // Save enrollment details if provided + detailQueries := osquery_utils.GetDetailQueries(appConfig) save := false if r, ok := hostDetails["os_version"]; ok { - detailQueries["os_version"].IngestFunc(svc.logger, host, []map[string]string{r}) + err := detailQueries["os_version"].IngestFunc(svc.logger, host, []map[string]string{r}) + if err != nil { + return "", errors.Wrap(err, "Ingesting os_version") + } save = true } if r, ok := hostDetails["osquery_info"]; ok { - detailQueries["osquery_info"].IngestFunc(svc.logger, host, []map[string]string{r}) + err := detailQueries["osquery_info"].IngestFunc(svc.logger, host, []map[string]string{r}) + if err != nil { + return "", errors.Wrap(err, "Ingesting osquery_info") + } save = true } if r, ok := hostDetails["system_info"]; ok { - detailQueries["system_info"].IngestFunc(svc.logger, host, []map[string]string{r}) + err := detailQueries["system_info"].IngestFunc(svc.logger, host, []map[string]string{r}) + if err != nil { + return "", errors.Wrap(err, "Ingesting system_info") + } save = true } if save { @@ -372,543 +375,6 @@ const hostAdditionalQueryPrefix = "fleet_additional_query_" // run from a distributed query campaign const hostDistributedQueryPrefix = "fleet_distributed_query_" -type detailQuery struct { - Query string - // Platforms is a list of platforms to run the query on. If this value is - // empty, run on all platforms. - Platforms []string - IngestFunc func(logger log.Logger, host *fleet.Host, rows []map[string]string) error -} - -// runForPlatform determines whether this detail query should run on the given platform -func (q *detailQuery) runForPlatform(platform string) bool { - if len(q.Platforms) == 0 { - return true - } - for _, p := range q.Platforms { - if p == platform { - return true - } - } - return false -} - -// detailQueries defines the detail queries that should be run on the host, as -// well as how the results of those queries should be ingested into the -// fleet.Host data model. This map should not be modified at runtime. -var detailQueries = map[string]detailQuery{ - "network_interface": { - Query: `select address, mac - from interface_details id join interface_addresses ia - on ia.interface = id.interface where length(mac) > 0 - order by (ibytes + obytes) desc`, - IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) (err error) { - if len(rows) == 0 { - logger.Log("component", "service", "method", "IngestFunc", "err", - "detail_query_network_interface expected 1 or more results") - return nil - } - - // Rows are ordered by traffic, so we will get the most active - // interface by iterating in order - var firstIPv4, firstIPv6 map[string]string - for _, row := range rows { - ip := net.ParseIP(row["address"]) - if ip == nil { - continue - } - - // Skip link-local and loopback interfaces - if ip.IsLinkLocalUnicast() || ip.IsLoopback() { - continue - } - - if strings.Contains(row["address"], ":") { - //IPv6 - if firstIPv6 == nil { - firstIPv6 = row - } - } else { - // IPv4 - if firstIPv4 == nil { - firstIPv4 = row - } - } - } - - var selected map[string]string - switch { - // Prefer IPv4 - case firstIPv4 != nil: - selected = firstIPv4 - // Otherwise IPv6 - case firstIPv6 != nil: - selected = firstIPv6 - // If only link-local and loopback found, still use the first - // interface so that we don't get an empty value. - default: - selected = rows[0] - } - - host.PrimaryIP = selected["address"] - host.PrimaryMac = selected["mac"] - return nil - }, - }, - "os_version": { - Query: "select * from os_version limit 1", - IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { - if len(rows) != 1 { - logger.Log("component", "service", "method", "IngestFunc", "err", - fmt.Sprintf("detail_query_os_version expected single result got %d", len(rows))) - return nil - } - - host.OSVersion = fmt.Sprintf( - "%s %s.%s.%s", - rows[0]["name"], - rows[0]["major"], - rows[0]["minor"], - rows[0]["patch"], - ) - host.OSVersion = strings.Trim(host.OSVersion, ".") - - if build, ok := rows[0]["build"]; ok { - host.Build = build - } - - host.Platform = rows[0]["platform"] - host.PlatformLike = rows[0]["platform_like"] - host.CodeName = rows[0]["code_name"] - - // On centos6 there is an osquery bug that leaves - // platform empty. Here we workaround. - if host.Platform == "" && - strings.Contains(strings.ToLower(rows[0]["name"]), "centos") { - host.Platform = "centos" - } - - return nil - }, - }, - "osquery_flags": { - // Collect the interval info (used for online status - // calculation) from the osquery flags. We typically control - // distributed_interval (but it's not required), and typically - // do not control config_tls_refresh. - Query: `select name, value from osquery_flags where name in ("distributed_interval", "config_tls_refresh", "config_refresh", "logger_tls_period")`, - IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { - var configTLSRefresh, configRefresh uint - var configRefreshSeen, configTLSRefreshSeen bool - for _, row := range rows { - switch row["name"] { - - case "distributed_interval": - interval, err := strconv.Atoi(emptyToZero(row["value"])) - if err != nil { - return errors.Wrap(err, "parsing distributed_interval") - } - host.DistributedInterval = uint(interval) - - case "config_tls_refresh": - // Prior to osquery 2.4.6, the flag was - // called `config_tls_refresh`. - interval, err := strconv.Atoi(emptyToZero(row["value"])) - if err != nil { - return errors.Wrap(err, "parsing config_tls_refresh") - } - configTLSRefresh = uint(interval) - configTLSRefreshSeen = true - - case "config_refresh": - // After 2.4.6 `config_tls_refresh` was - // aliased to `config_refresh`. - interval, err := strconv.Atoi(emptyToZero(row["value"])) - if err != nil { - return errors.Wrap(err, "parsing config_refresh") - } - configRefresh = uint(interval) - configRefreshSeen = true - - case "logger_tls_period": - interval, err := strconv.Atoi(emptyToZero(row["value"])) - if err != nil { - return errors.Wrap(err, "parsing logger_tls_period") - } - host.LoggerTLSPeriod = uint(interval) - } - } - - // Since the `config_refresh` flag existed prior to - // 2.4.6 and had a different meaning, we prefer - // `config_tls_refresh` if it was set, and use - // `config_refresh` as a fallback. - if configTLSRefreshSeen { - host.ConfigTLSRefresh = configTLSRefresh - } else if configRefreshSeen { - host.ConfigTLSRefresh = configRefresh - } - - return nil - }, - }, - "osquery_info": { - Query: "select * from osquery_info limit 1", - IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { - if len(rows) != 1 { - logger.Log("component", "service", "method", "IngestFunc", "err", - fmt.Sprintf("detail_query_osquery_info expected single result got %d", len(rows))) - return nil - } - - host.OsqueryVersion = rows[0]["version"] - - return nil - }, - }, - "system_info": { - Query: "select * from system_info limit 1", - IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { - if len(rows) != 1 { - logger.Log("component", "service", "method", "IngestFunc", "err", - fmt.Sprintf("detail_query_system_info expected single result got %d", len(rows))) - return nil - } - - var err error - host.Memory, err = strconv.ParseInt(emptyToZero(rows[0]["physical_memory"]), 10, 64) - if err != nil { - return err - } - host.Hostname = rows[0]["hostname"] - host.UUID = rows[0]["uuid"] - host.CPUType = rows[0]["cpu_type"] - host.CPUSubtype = rows[0]["cpu_subtype"] - host.CPUBrand = rows[0]["cpu_brand"] - host.CPUPhysicalCores, err = strconv.Atoi(emptyToZero(rows[0]["cpu_physical_cores"])) - if err != nil { - return err - } - host.CPULogicalCores, err = strconv.Atoi(emptyToZero(rows[0]["cpu_logical_cores"])) - if err != nil { - return err - } - host.HardwareVendor = rows[0]["hardware_vendor"] - host.HardwareModel = rows[0]["hardware_model"] - host.HardwareVersion = rows[0]["hardware_version"] - host.HardwareSerial = rows[0]["hardware_serial"] - host.ComputerName = rows[0]["computer_name"] - return nil - }, - }, - "uptime": { - Query: "select * from uptime limit 1", - IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { - if len(rows) != 1 { - logger.Log("component", "service", "method", "IngestFunc", "err", - fmt.Sprintf("detail_query_uptime expected single result got %d", len(rows))) - return nil - } - - uptimeSeconds, err := strconv.Atoi(emptyToZero(rows[0]["total_seconds"])) - if err != nil { - return err - } - host.Uptime = time.Duration(uptimeSeconds) * time.Second - - return nil - }, - }, - "software_macos": { - Query: ` -SELECT - name AS name, - bundle_short_version AS version, - 'Application (macOS)' AS type, - 'apps' AS source -FROM apps -UNION -SELECT - name AS name, - version AS version, - 'Package (Python)' AS type, - 'python_packages' AS source -FROM python_packages -UNION -SELECT - name AS name, - version AS version, - 'Browser plugin (Chrome)' AS type, - 'chrome_extensions' AS source -FROM chrome_extensions -UNION -SELECT - name AS name, - version AS version, - 'Browser plugin (Firefox)' AS type, - 'firefox_addons' AS source -FROM firefox_addons -UNION -SELECT - name As name, - version AS version, - 'Browser plugin (Safari)' AS type, - 'safari_extensions' AS source -FROM safari_extensions -UNION -SELECT - name AS name, - version AS version, - 'Package (Homebrew)' AS type, - 'homebrew_packages' AS source -FROM homebrew_packages; -`, - Platforms: []string{"darwin"}, - IngestFunc: ingestSoftware, - }, - "software_linux": { - Query: ` -SELECT - name AS name, - version AS version, - 'Package (deb)' AS type, - 'deb_packages' AS source -FROM deb_packages -UNION -SELECT - package AS name, - version AS version, - 'Package (Portage)' AS type, - 'portage_packages' AS source -FROM portage_packages -UNION -SELECT - name AS name, - version AS version, - 'Package (RPM)' AS type, - 'rpm_packages' AS source -FROM rpm_packages -UNION -SELECT - name AS name, - version AS version, - 'Package (NPM)' AS type, - 'npm_packages' AS source -FROM npm_packages -UNION -SELECT - name AS name, - version AS version, - 'Package (Atom)' AS type, - 'atom_packages' AS source -FROM atom_packages -UNION -SELECT - name AS name, - version AS version, - 'Package (Python)' AS type, - 'python_packages' AS source -FROM python_packages; -`, - Platforms: []string{"linux", "rhel", "ubuntu", "centos"}, - IngestFunc: ingestSoftware, - }, - "software_windows": { - Query: ` -SELECT - name AS name, - version AS version, - 'Program (Windows)' AS type, - 'programs' AS source -FROM programs -UNION -SELECT - name AS name, - version AS version, - 'Package (Python)' AS type, - 'python_packages' AS source -FROM python_packages -UNION -SELECT - name AS name, - version AS version, - 'Browser plugin (IE)' AS type, - 'ie_extensions' AS source -FROM ie_extensions -UNION -SELECT - name AS name, - version AS version, - 'Browser plugin (Chrome)' AS type, - 'chrome_extensions' AS source -FROM chrome_extensions -UNION -SELECT - name AS name, - version AS version, - 'Browser plugin (Firefox)' AS type, - 'firefox_addons' AS source -FROM firefox_addons -UNION -SELECT - name AS name, - version AS version, - 'Package (Chocolatey)' AS type, - 'chocolatey_packages' AS source -FROM chocolatey_packages -UNION -SELECT - name AS name, - version AS version, - 'Package (Atom)' AS type, - 'atom_packages' AS source -FROM atom_packages -UNION -SELECT - name AS name, - version AS version, - 'Package (Python)' AS type, - 'python_packages' AS source -FROM python_packages; -`, - Platforms: []string{"windows"}, - IngestFunc: ingestSoftware, - }, - "scheduled_query_stats": { - Query: ` - SELECT *, - (SELECT value from osquery_flags where name = 'pack_delimiter') AS delimiter - FROM osquery_schedule -`, - IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { - packs := map[string][]fleet.ScheduledQueryStats{} - - for _, row := range rows { - providedName := row["name"] - if providedName == "" { - level.Debug(logger).Log( - "msg", "host reported scheduled query with empty name", - "host", host.Hostname, - ) - continue - } - delimiter := row["delimiter"] - if delimiter == "" { - level.Debug(logger).Log( - "msg", "host reported scheduled query with empty delimiter", - "host", host.Hostname, - ) - continue - } - - // Split with a limit of 2 in case query name includes the - // delimiter. Not much we can do if pack name includes the - // delimiter. - trimmedName := strings.TrimPrefix(providedName, "pack"+delimiter) - parts := strings.SplitN(trimmedName, delimiter, 2) - if len(parts) != 2 { - level.Debug(logger).Log( - "msg", "could not split pack and query names", - "host", host.Hostname, - "name", providedName, - "delimiter", delimiter, - ) - continue - } - packName, scheduledName := parts[0], parts[1] - - stats := fleet.ScheduledQueryStats{ - ScheduledQueryName: scheduledName, - PackName: packName, - AverageMemory: cast.ToInt(row["average_memory"]), - Denylisted: cast.ToBool(row["denylisted"]), - Executions: cast.ToInt(row["executions"]), - Interval: cast.ToInt(row["interval"]), - // Cast to int first to allow cast.ToTime to interpret the unix timestamp. - LastExecuted: time.Unix(cast.ToInt64(row["last_executed"]), 0).UTC(), - OutputSize: cast.ToInt(row["output_size"]), - SystemTime: cast.ToInt(row["system_time"]), - UserTime: cast.ToInt(row["user_time"]), - WallTime: cast.ToInt(row["wall_time"]), - } - packs[packName] = append(packs[packName], stats) - } - - host.PackStats = []fleet.PackStats{} - for packName, stats := range packs { - host.PackStats = append( - host.PackStats, - fleet.PackStats{ - PackName: packName, - QueryStats: stats, - }, - ) - } - - return nil - }, - }, - "users": { - Query: `SELECT uid, username, type, groupname FROM users u JOIN groups g ON g.gid=u.gid;`, - IngestFunc: func(logger log.Logger, host *fleet.Host, rows []map[string]string) error { - var users []fleet.HostUser - for _, row := range rows { - uid, err := strconv.Atoi(row["uid"]) - if err != nil { - return errors.Wrapf(err, "converting uid %s to int", row["uid"]) - } - username := row["username"] - type_ := row["type"] - groupname := row["groupname"] - u := fleet.HostUser{ - Uid: uint(uid), - Username: username, - Type: type_, - GroupName: groupname, - } - users = append(users, u) - } - host.Users = users - - return nil - }, - }, -} - -func ingestSoftware(logger log.Logger, host *fleet.Host, rows []map[string]string) error { - software := fleet.HostSoftware{Modified: true} - - for _, row := range rows { - name := row["name"] - version := row["version"] - source := row["source"] - if name == "" { - level.Debug(logger).Log( - "msg", "host reported software with empty name", - "host", host.Hostname, - "version", version, - "source", source, - ) - continue - } - if source == "" { - level.Debug(logger).Log( - "msg", "host reported software with empty name", - "host", host.Hostname, - "version", version, - "name", name, - ) - continue - } - s := fleet.Software{Name: name, Version: version, Source: source} - software.Software = append(software.Software, s) - } - - host.HostSoftware = software - - return nil -} - // hostDetailQueries returns the map of queries that should be executed by // osqueryd to fill in the host details func (svc *Service) hostDetailQueries(host fleet.Host) (map[string]string, error) { @@ -917,25 +383,19 @@ func (svc *Service) hostDetailQueries(host fleet.Host) (map[string]string, error // No need to update already fresh details return queries, nil } + config, err := svc.ds.AppConfig() + if err != nil { + return nil, osqueryError{message: "get additional queries: " + err.Error()} + } + detailQueries := osquery_utils.GetDetailQueries(config) for name, query := range detailQueries { - if query.runForPlatform(host.Platform) { - if strings.HasPrefix(name, "software_") { - // Feature flag this because of as-yet-untested performance - // considerations. - if os.Getenv("FLEET_BETA_SOFTWARE_INVENTORY") == "" { - continue - } - } + if query.RunsForPlatform(host.Platform) { queries[hostDetailQueryPrefix+name] = query.Query } } // Get additional queries - config, err := svc.ds.AppConfig() - if err != nil { - return nil, osqueryError{message: "get additional queries: " + err.Error()} - } if config.AdditionalQueries == nil { // No additional queries set return queries, nil @@ -1004,12 +464,19 @@ func (svc *Service) GetDistributedQueries(ctx context.Context) (map[string]strin // provided fleet.Host appropriately. func (svc *Service) ingestDetailQuery(host *fleet.Host, name string, rows []map[string]string) error { trimmedQuery := strings.TrimPrefix(name, hostDetailQueryPrefix) + + config, err := svc.ds.AppConfig() + if err != nil { + return osqueryError{message: "ingest detail query: " + err.Error()} + } + + detailQueries := osquery_utils.GetDetailQueries(config) query, ok := detailQueries[trimmedQuery] if !ok { return osqueryError{message: "unknown detail query " + trimmedQuery} } - err := query.IngestFunc(svc.logger, host, rows) + err = query.IngestFunc(svc.logger, host, rows) if err != nil { return osqueryError{ message: fmt.Sprintf("ingesting query %s: %s", name, err.Error()), @@ -1025,7 +492,7 @@ func (svc *Service) ingestDetailQuery(host *fleet.Host, name string, rows []map[ // ingestLabelQuery records the results of label queries run by a host func (svc *Service) ingestLabelQuery(host fleet.Host, query string, rows []map[string]string, results map[uint]bool) error { trimmedQuery := strings.TrimPrefix(query, hostLabelQueryPrefix) - trimmedQueryNum, err := strconv.Atoi(emptyToZero(trimmedQuery)) + trimmedQueryNum, err := strconv.Atoi(osquery_utils.EmptyToZero(trimmedQuery)) if err != nil { return errors.Wrap(err, "converting query from string to int") } @@ -1040,7 +507,7 @@ func (svc *Service) ingestLabelQuery(host fleet.Host, query string, rows []map[s func (svc *Service) ingestDistributedQuery(host fleet.Host, name string, rows []map[string]string, failed bool, errMsg string) error { trimmedQuery := strings.TrimPrefix(name, hostDistributedQueryPrefix) - campaignID, err := strconv.Atoi(emptyToZero(trimmedQuery)) + campaignID, err := strconv.Atoi(osquery_utils.EmptyToZero(trimmedQuery)) if err != nil { return osqueryError{message: "unable to parse campaign ID: " + trimmedQuery} } diff --git a/server/service/service_osquery_test.go b/server/service/service_osquery_test.go index d106cc5512..40bfa51456 100644 --- a/server/service/service_osquery_test.go +++ b/server/service/service_osquery_test.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "sort" "strconv" "strings" "sync" @@ -25,14 +24,14 @@ import ( "github.com/fleetdm/fleet/v4/server/mock" "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/pubsub" + "github.com/fleetdm/fleet/v4/server/service/osquery_utils" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -// 3 detail queries are currently feature flagged off by default. -var expectedDetailQueries = len(detailQueries) - 3 +var expectedDetailQueries = len(osquery_utils.GetDetailQueries(&fleet.AppConfig{EnableHostUsers: true})) func TestEnrollAgent(t *testing.T) { ds := new(mock.Store) @@ -50,6 +49,9 @@ func TestEnrollAgent(t *testing.T) { OsqueryHostID: osqueryHostId, NodeKey: nodeKey, }, nil } + ds.AppConfigFunc = func() (*fleet.AppConfig, error) { + return &fleet.AppConfig{EnableHostUsers: true}, nil + } svc := newTestService(ds, nil, nil) @@ -91,6 +93,9 @@ func TestEnrollAgentDetails(t *testing.T) { gotHost = host return nil } + ds.AppConfigFunc = func() (*fleet.AppConfig, error) { + return &fleet.AppConfig{EnableHostUsers: true}, nil + } svc := newTestService(ds, nil, nil) @@ -246,7 +251,7 @@ func TestHostDetailQueries(t *testing.T) { ds := new(mock.Store) additional := json.RawMessage(`{"foobar": "select foo", "bim": "bam"}`) ds.AppConfigFunc = func() (*fleet.AppConfig, error) { - return &fleet.AppConfig{AdditionalQueries: &additional}, nil + return &fleet.AppConfig{AdditionalQueries: &additional, EnableHostUsers: true}, nil } mockClock := clock.NewMockClock() @@ -324,7 +329,7 @@ func TestLabelQueries(t *testing.T) { return nil } ds.AppConfigFunc = func() (*fleet.AppConfig, error) { - return &fleet.AppConfig{}, nil + return &fleet.AppConfig{EnableHostUsers: true}, nil } lq.On("QueriesForHost", uint(0)).Return(map[string]string{}, nil) @@ -524,7 +529,7 @@ func TestDetailQueriesWithEmptyStrings(t *testing.T) { ctx := hostctx.NewContext(context.Background(), host) ds.AppConfigFunc = func() (*fleet.AppConfig, error) { - return &fleet.AppConfig{}, nil + return &fleet.AppConfig{EnableHostUsers: true}, nil } ds.LabelQueriesForHostFunc = func(*fleet.Host, time.Time) (map[string]string, error) { return map[string]string{}, nil @@ -701,7 +706,7 @@ func TestDetailQueries(t *testing.T) { lq.On("QueriesForHost", host.ID).Return(map[string]string{}, nil) ds.AppConfigFunc = func() (*fleet.AppConfig, error) { - return &fleet.AppConfig{}, nil + return &fleet.AppConfig{EnableHostUsers: true}, nil } ds.LabelQueriesForHostFunc = func(*fleet.Host, time.Time) (map[string]string, error) { return map[string]string{}, nil @@ -886,259 +891,6 @@ func TestDetailQueries(t *testing.T) { assert.Zero(t, acc) } -func TestDetailQueryNetworkInterfaces(t *testing.T) { - var initialHost fleet.Host - host := initialHost - - ingest := detailQueries["network_interface"].IngestFunc - - assert.NoError(t, ingest(log.NewNopLogger(), &host, nil)) - assert.Equal(t, initialHost, host) - - var rows []map[string]string - require.NoError(t, json.Unmarshal([]byte(` -[ - {"address":"127.0.0.1","mac":"00:00:00:00:00:00"}, - {"address":"::1","mac":"00:00:00:00:00:00"}, - {"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"}, - {"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"}, - {"address":"192.168.1.3","mac":"f4:5d:79:93:58:5b"}, - {"address":"fe80::241a:9aff:fe60:d80a%awdl0","mac":"27:1b:aa:60:e8:0a"}, - {"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"} -]`), - &rows, - )) - - assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) - assert.Equal(t, "192.168.1.3", host.PrimaryIP) - assert.Equal(t, "f4:5d:79:93:58:5b", host.PrimaryMac) - - // Only IPv6 - require.NoError(t, json.Unmarshal([]byte(` -[ - {"address":"127.0.0.1","mac":"00:00:00:00:00:00"}, - {"address":"::1","mac":"00:00:00:00:00:00"}, - {"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"}, - {"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"}, - {"address":"2604:3f08:1337:9411:cbe:814f:51a6:e4e3","mac":"27:1b:aa:60:e8:0a"}, - {"address":"3333:3f08:1337:9411:cbe:814f:51a6:e4e3","mac":"bb:1b:aa:60:e8:bb"}, - {"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"} -]`), - &rows, - )) - - assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) - assert.Equal(t, "2604:3f08:1337:9411:cbe:814f:51a6:e4e3", host.PrimaryIP) - assert.Equal(t, "27:1b:aa:60:e8:0a", host.PrimaryMac) - - // IPv6 appears before IPv4 (v4 should be prioritized) - require.NoError(t, json.Unmarshal([]byte(` -[ - {"address":"127.0.0.1","mac":"00:00:00:00:00:00"}, - {"address":"::1","mac":"00:00:00:00:00:00"}, - {"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"}, - {"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"}, - {"address":"2604:3f08:1337:9411:cbe:814f:51a6:e4e3","mac":"27:1b:aa:60:e8:0a"}, - {"address":"205.111.43.79","mac":"ab:1b:aa:60:e8:0a"}, - {"address":"205.111.44.80","mac":"bb:bb:aa:60:e8:0a"}, - {"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"} -]`), - &rows, - )) - - assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) - assert.Equal(t, "205.111.43.79", host.PrimaryIP) - assert.Equal(t, "ab:1b:aa:60:e8:0a", host.PrimaryMac) - - // Only link-local/loopback - require.NoError(t, json.Unmarshal([]byte(` -[ - {"address":"127.0.0.1","mac":"00:00:00:00:00:00"}, - {"address":"::1","mac":"00:00:00:00:00:00"}, - {"address":"fe80::1%lo0","mac":"00:00:00:00:00:00"}, - {"address":"fe80::df:429b:971c:d051%en0","mac":"f4:5c:89:92:57:5b"}, - {"address":"fe80::241a:9aff:fe60:d80a%awdl0","mac":"27:1b:aa:60:e8:0a"}, - {"address":"fe80::3a6f:582f:86c5:8296%utun0","mac":"00:00:00:00:00:00"} -]`), - &rows, - )) - - assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) - assert.Equal(t, "127.0.0.1", host.PrimaryIP) - assert.Equal(t, "00:00:00:00:00:00", host.PrimaryMac) -} - -func TestDetailQueryScheduledQueryStats(t *testing.T) { - host := fleet.Host{} - - ingest := detailQueries["scheduled_query_stats"].IngestFunc - - assert.NoError(t, ingest(log.NewNopLogger(), &host, nil)) - assert.Len(t, host.PackStats, 0) - - resJSON := ` -[ - { - "average_memory":"33", - "delimiter":"/", - "denylisted":"0", - "executions":"1", - "interval":"33", - "last_executed":"1620325191", - "name":"pack/pack-2/time", - "output_size":"", - "query":"SELECT * FROM time", - "system_time":"100", - "user_time":"60", - "wall_time":"180" - }, - { - "average_memory":"8000", - "delimiter":"/", - "denylisted":"0", - "executions":"164", - "interval":"30", - "last_executed":"1620325191", - "name":"pack/test/osquery info", - "output_size":"1337", - "query":"SELECT * FROM osquery_info", - "system_time":"150", - "user_time":"180", - "wall_time":"0" - }, - { - "average_memory":"50400", - "delimiter":"/", - "denylisted":"1", - "executions":"188", - "interval":"30", - "last_executed":"1620325203", - "name":"pack/test/processes?", - "output_size":"", - "query":"SELECT * FROM processes", - "system_time":"140", - "user_time":"190", - "wall_time":"1" - }, - { - "average_memory":"0", - "delimiter":"/", - "denylisted":"0", - "executions":"1", - "interval":"3600", - "last_executed":"1620323381", - "name":"pack/test/processes?-1", - "output_size":"", - "query":"SELECT * FROM processes", - "system_time":"0", - "user_time":"0", - "wall_time":"0" - }, - { - "average_memory":"0", - "delimiter":"/", - "denylisted":"0", - "executions":"105", - "interval":"47", - "last_executed":"1620325190", - "name":"pack/test/time", - "output_size":"", - "query":"SELECT * FROM time", - "system_time":"70", - "user_time":"50", - "wall_time":"1" - } -] -` - - var rows []map[string]string - require.NoError(t, json.Unmarshal([]byte(resJSON), &rows)) - - assert.NoError(t, ingest(log.NewNopLogger(), &host, rows)) - assert.Len(t, host.PackStats, 2) - sort.Slice(host.PackStats, func(i, j int) bool { - return host.PackStats[i].PackName < host.PackStats[j].PackName - }) - assert.Equal(t, host.PackStats[0].PackName, "pack-2") - assert.ElementsMatch(t, host.PackStats[0].QueryStats, - []fleet.ScheduledQueryStats{ - { - ScheduledQueryName: "time", - PackName: "pack-2", - AverageMemory: 33, - Denylisted: false, - Executions: 1, - Interval: 33, - LastExecuted: time.Unix(1620325191, 0).UTC(), - OutputSize: 0, - SystemTime: 100, - UserTime: 60, - WallTime: 180, - }, - }, - ) - assert.Equal(t, host.PackStats[1].PackName, "test") - assert.ElementsMatch(t, host.PackStats[1].QueryStats, - []fleet.ScheduledQueryStats{ - { - ScheduledQueryName: "osquery info", - PackName: "test", - AverageMemory: 8000, - Denylisted: false, - Executions: 164, - Interval: 30, - LastExecuted: time.Unix(1620325191, 0).UTC(), - OutputSize: 1337, - SystemTime: 150, - UserTime: 180, - WallTime: 0, - }, - { - ScheduledQueryName: "processes?", - PackName: "test", - AverageMemory: 50400, - Denylisted: true, - Executions: 188, - Interval: 30, - LastExecuted: time.Unix(1620325203, 0).UTC(), - OutputSize: 0, - SystemTime: 140, - UserTime: 190, - WallTime: 1, - }, - { - ScheduledQueryName: "processes?-1", - PackName: "test", - AverageMemory: 0, - Denylisted: false, - Executions: 1, - Interval: 3600, - LastExecuted: time.Unix(1620323381, 0).UTC(), - OutputSize: 0, - SystemTime: 0, - UserTime: 0, - WallTime: 0, - }, - { - ScheduledQueryName: "time", - PackName: "test", - AverageMemory: 0, - Denylisted: false, - Executions: 105, - Interval: 47, - LastExecuted: time.Unix(1620325190, 0).UTC(), - OutputSize: 0, - SystemTime: 70, - UserTime: 50, - WallTime: 1, - }, - }, - ) - - assert.NoError(t, ingest(log.NewNopLogger(), &host, nil)) - assert.Len(t, host.PackStats, 0) -} - func TestNewDistributedQueryCampaign(t *testing.T) { ds := &mock.Store{ AppConfigStore: mock.AppConfigStore{ @@ -1233,7 +985,7 @@ func TestDistributedQueryResults(t *testing.T) { return nil } ds.AppConfigFunc = func() (*fleet.AppConfig, error) { - return &fleet.AppConfig{}, nil + return &fleet.AppConfig{EnableHostUsers: true}, nil } host := &fleet.Host{ID: 1, Platform: "windows"} @@ -1882,6 +1634,9 @@ func TestDistributedQueriesReloadsHostIfDetailsAreIn(t *testing.T) { require.Equal(t, uint(42), id) return &fleet.Host{ID: 42, Platform: "darwin", PrimaryIP: ip}, nil } + ds.AppConfigFunc = func() (*fleet.AppConfig, error) { + return &fleet.AppConfig{EnableHostUsers: true}, nil + } ctx := hostctx.NewContext(context.Background(), *host) diff --git a/server/service/testing_utils.go b/server/service/testing_utils.go index 2f61737c99..9ba014b0e1 100644 --- a/server/service/testing_utils.go +++ b/server/service/testing_utils.go @@ -83,27 +83,6 @@ func newTestServiceWithClock(ds fleet.Datastore, rs fleet.QueryResultStore, lq f return svc } -func createTestAppConfig(t *testing.T, ds fleet.Datastore) *fleet.AppConfig { - config := &fleet.AppConfig{ - OrgName: "Tyrell Corp", - OrgLogoURL: "https://tyrell.com/image.png", - ServerURL: "https://fleet.tyrell.com", - SMTPConfigured: true, - SMTPSenderAddress: "test@example.com", - SMTPServer: "smtp.tyrell.com", - SMTPPort: 587, - SMTPAuthenticationType: fleet.AuthTypeUserNamePassword, - SMTPUserName: "deckard", - SMTPPassword: "replicant", - SMTPVerifySSLCerts: true, - SMTPEnableTLS: true, - } - result, err := ds.NewAppConfig(config) - require.Nil(t, err) - require.NotNil(t, result) - return result -} - func createTestUsers(t *testing.T, ds fleet.Datastore) map[string]fleet.User { users := make(map[string]fleet.User) for _, u := range testUsers {