diff --git a/changes/allow-disabling-scheduled-query-stats b/changes/allow-disabling-scheduled-query-stats new file mode 100644 index 0000000000..d5db86b709 --- /dev/null +++ b/changes/allow-disabling-scheduled-query-stats @@ -0,0 +1 @@ +* Allow disabling scheduled query stats via app config diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index 0ac631f441..b8466ff645 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -398,6 +398,7 @@ spec: host_expiry_window: 0 host_settings: enable_host_users: true + enable_scheduled_query_stats: false enable_software_inventory: false org_info: org_logo_url: "" @@ -449,7 +450,7 @@ spec: enable_vulnerabilities_webhook: false host_batch_size: 0 ` - expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"}}} + expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false,"enable_scheduled_query_stats":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"}}} ` assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "config"})) @@ -467,6 +468,7 @@ spec: host_expiry_window: 0 host_settings: enable_host_users: true + enable_scheduled_query_stats: false enable_software_inventory: false license: expiration: "0001-01-01T00:00:00Z" @@ -548,7 +550,7 @@ spec: enable_vulnerabilities_webhook: false host_batch_size: 0 ` - expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"},"update_interval":{"osquery_detail":3600000000000,"osquery_policy":3600000000000},"vulnerabilities":{"databases_path":"","periodicity":0,"cpe_database_url":"","cve_feed_prefix_url":"","current_instance_checks":"","disable_data_sync":false},"license":{"tier":"free","expiration":"0001-01-01T00:00:00Z"},"logging":{"debug":true,"json":false,"result":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}},"status":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}}}}} + expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false,"enable_scheduled_query_stats":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"},"update_interval":{"osquery_detail":3600000000000,"osquery_policy":3600000000000},"vulnerabilities":{"databases_path":"","periodicity":0,"cpe_database_url":"","cve_feed_prefix_url":"","current_instance_checks":"","disable_data_sync":false},"license":{"tier":"free","expiration":"0001-01-01T00:00:00Z"},"logging":{"debug":true,"json":false,"result":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}},"status":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}}}}} ` assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "config", "--include-server-config"})) diff --git a/docs/01-Using-Fleet/configuration-files/README.md b/docs/01-Using-Fleet/configuration-files/README.md index a5abdb15ff..3d28273ddd 100644 --- a/docs/01-Using-Fleet/configuration-files/README.md +++ b/docs/01-Using-Fleet/configuration-files/README.md @@ -6,6 +6,7 @@ - [Enroll secrets](#enroll-secrets) - [Teams](#teams) - [Organization settings](#organization-settings) +- [Host settings](#host-settings) Entities in Fleet, such as queries, packs, labels, agent options, and enroll secrets, can be managed with configuration files in yaml syntax. @@ -491,3 +492,12 @@ spec: WARNING: this will log potentially a lot of data. Some of that data might be private, please verify it before posting it in a public channel or a Github issue. + +### Host settings + +The `host_settings` section of the configuration yaml allows to define what predefined queries are sent to the hosts and +later on processed by Fleet for different functionalities. + +- `host_settings.enable_host_users`: boolean value that when enabled Fleet will send the query needed to gather user data +- `host_settings.enable_software_inventory`: boolean value that when enabled Fleet will send the query needed to gather the list of software installed along with other metadata +- `host_settings.enable_scheduled_query_stats`: boolean value that when enabled Fleet will send the query needed to gather statistics about query executions in a host diff --git a/server/datastore/mysql/app_configs_test.go b/server/datastore/mysql/app_configs_test.go index 8959912dad..d1d73a3fb7 100644 --- a/server/datastore/mysql/app_configs_test.go +++ b/server/datastore/mysql/app_configs_test.go @@ -280,6 +280,7 @@ func testAppConfigDefaults(t *testing.T, ds *Datastore) { require.Equal(t, 24*time.Hour, ac.WebhookSettings.Interval.Duration) require.False(t, ac.WebhookSettings.HostStatusWebhook.Enable) require.True(t, ac.HostSettings.EnableHostUsers) + require.True(t, ac.HostSettings.EnableScheduledQueryStats) require.False(t, ac.HostSettings.EnableSoftwareInventory) _, err = ds.writer.Exec( @@ -293,5 +294,6 @@ func testAppConfigDefaults(t *testing.T, ds *Datastore) { require.Equal(t, 12*time.Hour, ac.WebhookSettings.Interval.Duration) require.False(t, ac.HostSettings.EnableHostUsers) + require.True(t, ac.HostSettings.EnableScheduledQueryStats) require.False(t, ac.HostSettings.EnableSoftwareInventory) } diff --git a/server/fleet/app.go b/server/fleet/app.go index 527c7213ce..f81e331e78 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -230,6 +230,7 @@ func (c *AppConfig) ApplyDefaultsForNewInstalls() { func (c *AppConfig) ApplyDefaults() { c.HostSettings.EnableHostUsers = true + c.HostSettings.EnableScheduledQueryStats = true c.WebhookSettings.Interval.Duration = 24 * time.Hour } @@ -255,9 +256,10 @@ type HostExpirySettings struct { } type HostSettings struct { - EnableHostUsers bool `json:"enable_host_users"` - EnableSoftwareInventory bool `json:"enable_software_inventory"` - AdditionalQueries *json.RawMessage `json:"additional_queries,omitempty"` + EnableHostUsers bool `json:"enable_host_users"` + EnableSoftwareInventory bool `json:"enable_software_inventory"` + EnableScheduledQueryStats bool `json:"enable_scheduled_query_stats"` + AdditionalQueries *json.RawMessage `json:"additional_queries,omitempty"` } type OrderDirection int diff --git a/server/service/osquery_utils/queries.go b/server/service/osquery_utils/queries.go index b3093be38d..d5ab0d6164 100644 --- a/server/service/osquery_utils/queries.go +++ b/server/service/osquery_utils/queries.go @@ -267,13 +267,6 @@ var detailQueries = map[string]DetailQuery{ return nil }, }, - "scheduled_query_stats": { - Query: ` - SELECT *, - (SELECT value from osquery_flags where name = 'pack_delimiter') AS delimiter - FROM osquery_schedule`, - DirectIngestFunc: directIngestScheduledQueryStats, - }, "disk_space_unix": { Query: ` SELECT (blocks_available * 100 / blocks) AS percent_disk_space_available, @@ -498,6 +491,14 @@ FROM python_packages; DirectIngestFunc: directIngestSoftware, } +var scheduledQueryStats = DetailQuery{ + Query: ` + SELECT *, + (SELECT value from osquery_flags where name = 'pack_delimiter') AS delimiter + FROM osquery_schedule`, + DirectIngestFunc: directIngestScheduledQueryStats, +} + var usersQuery = DetailQuery{ // Note we use the cached_groups CTE (`WITH` clause) here to suggest to SQLite that it generate // the `groups` table only once. Without doing this, on some Windows systems (Domain Controllers) @@ -753,5 +754,9 @@ func GetDetailQueries(ac *fleet.AppConfig) map[string]DetailQuery { generatedMap["users"] = usersQuery } + if ac != nil && ac.HostSettings.EnableScheduledQueryStats { + generatedMap["scheduled_query_stats"] = scheduledQueryStats + } + return generatedMap } diff --git a/server/service/osquery_utils/queries_test.go b/server/service/osquery_utils/queries_test.go index 376f1067cf..13c823b72d 100644 --- a/server/service/osquery_utils/queries_test.go +++ b/server/service/osquery_utils/queries_test.go @@ -110,7 +110,7 @@ func TestDetailQueryScheduledQueryStats(t *testing.T) { return nil } - ingest := GetDetailQueries(nil)["scheduled_query_stats"].DirectIngestFunc + ingest := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableScheduledQueryStats: true}})["scheduled_query_stats"].DirectIngestFunc ctx := context.Background() assert.NoError(t, ingest(ctx, log.NewNopLogger(), &host, ds, nil, false)) @@ -289,13 +289,12 @@ func sortedKeysCompare(t *testing.T, m map[string]DetailQuery, expectedKeys []st func TestGetDetailQueries(t *testing.T) { queriesNoConfig := GetDetailQueries(nil) - require.Len(t, queriesNoConfig, 12) + require.Len(t, queriesNoConfig, 11) baseQueries := []string{ "network_interface", "os_version", "osquery_flags", "osquery_info", - "scheduled_query_stats", "system_info", "uptime", "disk_space_unix", @@ -306,14 +305,14 @@ func TestGetDetailQueries(t *testing.T) { } sortedKeysCompare(t, queriesNoConfig, baseQueries) - queriesWithUsers := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}}) + queriesWithUsers := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true, EnableScheduledQueryStats: true}}) require.Len(t, queriesWithUsers, 13) - sortedKeysCompare(t, queriesWithUsers, append(baseQueries, "users")) + sortedKeysCompare(t, queriesWithUsers, append(baseQueries, "users", "scheduled_query_stats")) - queriesWithUsersAndSoftware := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true, EnableSoftwareInventory: true}}) + queriesWithUsersAndSoftware := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true, EnableSoftwareInventory: true, EnableScheduledQueryStats: true}}) require.Len(t, queriesWithUsersAndSoftware, 16) sortedKeysCompare(t, queriesWithUsersAndSoftware, - append(baseQueries, "users", "software_macos", "software_linux", "software_windows")) + append(baseQueries, "users", "software_macos", "software_linux", "software_windows", "scheduled_query_stats")) } func TestDirectIngestMDM(t *testing.T) {