Allow disabling scheduled query stats via app config (#4049)

* Allow disabling scheduled query stats via app config

* Update tests

* Fix test

* Moar test fixes

* Remove redundant set

* Add documentation

* Fix typo in docs

changes/allow-disabling-scheduled-query-stats

@@ -0,0 +1 @@
+* Allow disabling scheduled query stats via app config

cmd/fleetctl/get_test.go

@@ -398,6 +398,7 @@ spec:
     host_expiry_window: 0
   host_settings:
     enable_host_users: true
+    enable_scheduled_query_stats: false
     enable_software_inventory: false
   org_info:
     org_logo_url: ""
@@ -449,7 +450,7 @@ spec:
       enable_vulnerabilities_webhook: false
       host_batch_size: 0
 `
-		expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"}}}
+		expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false,"enable_scheduled_query_stats":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"}}}
 `
 
 		assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "config"}))
@@ -467,6 +468,7 @@ spec:
     host_expiry_window: 0
   host_settings:
     enable_host_users: true
+    enable_scheduled_query_stats: false
     enable_software_inventory: false
   license:
     expiration: "0001-01-01T00:00:00Z"
@@ -548,7 +550,7 @@ spec:
       enable_vulnerabilities_webhook: false
       host_batch_size: 0
 `
-		expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"},"update_interval":{"osquery_detail":3600000000000,"osquery_policy":3600000000000},"vulnerabilities":{"databases_path":"","periodicity":0,"cpe_database_url":"","cve_feed_prefix_url":"","current_instance_checks":"","disable_data_sync":false},"license":{"tier":"free","expiration":"0001-01-01T00:00:00Z"},"logging":{"debug":true,"json":false,"result":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}},"status":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}}}}}
+		expectedJson := `{"kind":"config","apiVersion":"v1","spec":{"org_info":{"org_name":"","org_logo_url":""},"server_settings":{"server_url":"","live_query_disabled":false,"enable_analytics":false,"deferred_save_host":false},"smtp_settings":{"enable_smtp":false,"configured":false,"sender_address":"","server":"","port":0,"authentication_type":"","user_name":"","password":"","enable_ssl_tls":false,"authentication_method":"","domain":"","verify_ssl_certs":false,"enable_start_tls":false},"host_expiry_settings":{"host_expiry_enabled":false,"host_expiry_window":0},"host_settings":{"enable_host_users":true,"enable_software_inventory":false,"enable_scheduled_query_stats":false},"sso_settings":{"entity_id":"","issuer_uri":"","idp_image_url":"","metadata":"","metadata_url":"","idp_name":"","enable_sso":false,"enable_sso_idp_login":false},"vulnerability_settings":{"databases_path":"/some/path"},"webhook_settings":{"host_status_webhook":{"enable_host_status_webhook":false,"destination_url":"","host_percentage":0,"days_count":0},"failing_policies_webhook":{"enable_failing_policies_webhook":false,"destination_url":"","policy_ids":null,"host_batch_size":0},"vulnerabilities_webhook":{"enable_vulnerabilities_webhook":false,"destination_url":"","host_batch_size":0},"interval":"0s"},"update_interval":{"osquery_detail":3600000000000,"osquery_policy":3600000000000},"vulnerabilities":{"databases_path":"","periodicity":0,"cpe_database_url":"","cve_feed_prefix_url":"","current_instance_checks":"","disable_data_sync":false},"license":{"tier":"free","expiration":"0001-01-01T00:00:00Z"},"logging":{"debug":true,"json":false,"result":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}},"status":{"plugin":"filesystem","config":{"enable_log_compression":false,"enable_log_rotation":false,"result_log_file":"/dev/null","status_log_file":"/dev/null"}}}}}
 `
 
 		assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "config", "--include-server-config"}))

docs/01-Using-Fleet/configuration-files/README.md

@@ -6,6 +6,7 @@
 - [Enroll secrets](#enroll-secrets)
 - [Teams](#teams)
 - [Organization settings](#organization-settings)
+- [Host settings](#host-settings)
 
 Entities in Fleet, such as queries, packs, labels, agent options, and enroll secrets, can be managed with configuration files in yaml syntax.
 
@@ -491,3 +492,12 @@ spec:
 
 WARNING: this will log potentially a lot of data. Some of that data might be private, please verify it before posting it
 in a public channel or a Github issue.
+
+### Host settings
+
+The `host_settings` section of the configuration yaml allows to define what predefined queries are sent to the hosts and 
+later on processed by Fleet for different functionalities.
+
+- `host_settings.enable_host_users`: boolean value that when enabled Fleet will send the query needed to gather user data
+- `host_settings.enable_software_inventory`: boolean value that when enabled Fleet will send the query needed to gather the list of software installed along with other metadata
+- `host_settings.enable_scheduled_query_stats`: boolean value that when enabled Fleet will send the query needed to gather statistics about query executions in a host

server/datastore/mysql/app_configs_test.go

@@ -280,6 +280,7 @@ func testAppConfigDefaults(t *testing.T, ds *Datastore) {
 	require.Equal(t, 24*time.Hour, ac.WebhookSettings.Interval.Duration)
 	require.False(t, ac.WebhookSettings.HostStatusWebhook.Enable)
 	require.True(t, ac.HostSettings.EnableHostUsers)
+	require.True(t, ac.HostSettings.EnableScheduledQueryStats)
 	require.False(t, ac.HostSettings.EnableSoftwareInventory)
 
 	_, err = ds.writer.Exec(
@@ -293,5 +294,6 @@ func testAppConfigDefaults(t *testing.T, ds *Datastore) {
 
 	require.Equal(t, 12*time.Hour, ac.WebhookSettings.Interval.Duration)
 	require.False(t, ac.HostSettings.EnableHostUsers)
+	require.True(t, ac.HostSettings.EnableScheduledQueryStats)
 	require.False(t, ac.HostSettings.EnableSoftwareInventory)
 }

server/fleet/app.go

@@ -230,6 +230,7 @@ func (c *AppConfig) ApplyDefaultsForNewInstalls() {
 
 func (c *AppConfig) ApplyDefaults() {
 	c.HostSettings.EnableHostUsers = true
+	c.HostSettings.EnableScheduledQueryStats = true
 	c.WebhookSettings.Interval.Duration = 24 * time.Hour
 }
 
@@ -255,9 +256,10 @@ type HostExpirySettings struct {
 }
 
 type HostSettings struct {
-	EnableHostUsers         bool             `json:"enable_host_users"`
-	EnableSoftwareInventory bool             `json:"enable_software_inventory"`
-	AdditionalQueries       *json.RawMessage `json:"additional_queries,omitempty"`
+	EnableHostUsers           bool             `json:"enable_host_users"`
+	EnableSoftwareInventory   bool             `json:"enable_software_inventory"`
+	EnableScheduledQueryStats bool             `json:"enable_scheduled_query_stats"`
+	AdditionalQueries         *json.RawMessage `json:"additional_queries,omitempty"`
 }
 
 type OrderDirection int

server/service/osquery_utils/queries.go

@@ -267,13 +267,6 @@ var detailQueries = map[string]DetailQuery{
 			return nil
 		},
 	},
-	"scheduled_query_stats": {
-		Query: `
-			SELECT *,
-				(SELECT value from osquery_flags where name = 'pack_delimiter') AS delimiter
-			FROM osquery_schedule`,
-		DirectIngestFunc: directIngestScheduledQueryStats,
-	},
 	"disk_space_unix": {
 		Query: `
 SELECT (blocks_available * 100 / blocks) AS percent_disk_space_available,
@@ -498,6 +491,14 @@ FROM python_packages;
 	DirectIngestFunc: directIngestSoftware,
 }
 
+var scheduledQueryStats = DetailQuery{
+	Query: `
+			SELECT *,
+				(SELECT value from osquery_flags where name = 'pack_delimiter') AS delimiter
+			FROM osquery_schedule`,
+	DirectIngestFunc: directIngestScheduledQueryStats,
+}
+
 var usersQuery = DetailQuery{
 	// Note we use the cached_groups CTE (`WITH` clause) here to suggest to SQLite that it generate
 	// the `groups` table only once. Without doing this, on some Windows systems (Domain Controllers)
@@ -753,5 +754,9 @@ func GetDetailQueries(ac *fleet.AppConfig) map[string]DetailQuery {
 		generatedMap["users"] = usersQuery
 	}
 
+	if ac != nil && ac.HostSettings.EnableScheduledQueryStats {
+		generatedMap["scheduled_query_stats"] = scheduledQueryStats
+	}
+
 	return generatedMap
 }

server/service/osquery_utils/queries_test.go

@@ -110,7 +110,7 @@ func TestDetailQueryScheduledQueryStats(t *testing.T) {
 		return nil
 	}
 
-	ingest := GetDetailQueries(nil)["scheduled_query_stats"].DirectIngestFunc
+	ingest := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableScheduledQueryStats: true}})["scheduled_query_stats"].DirectIngestFunc
 
 	ctx := context.Background()
 	assert.NoError(t, ingest(ctx, log.NewNopLogger(), &host, ds, nil, false))
@@ -289,13 +289,12 @@ func sortedKeysCompare(t *testing.T, m map[string]DetailQuery, expectedKeys []st
 
 func TestGetDetailQueries(t *testing.T) {
 	queriesNoConfig := GetDetailQueries(nil)
-	require.Len(t, queriesNoConfig, 12)
+	require.Len(t, queriesNoConfig, 11)
 	baseQueries := []string{
 		"network_interface",
 		"os_version",
 		"osquery_flags",
 		"osquery_info",
-		"scheduled_query_stats",
 		"system_info",
 		"uptime",
 		"disk_space_unix",
@@ -306,14 +305,14 @@ func TestGetDetailQueries(t *testing.T) {
 	}
 	sortedKeysCompare(t, queriesNoConfig, baseQueries)
 
-	queriesWithUsers := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true}})
+	queriesWithUsers := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true, EnableScheduledQueryStats: true}})
 	require.Len(t, queriesWithUsers, 13)
-	sortedKeysCompare(t, queriesWithUsers, append(baseQueries, "users"))
+	sortedKeysCompare(t, queriesWithUsers, append(baseQueries, "users", "scheduled_query_stats"))
 
-	queriesWithUsersAndSoftware := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true, EnableSoftwareInventory: true}})
+	queriesWithUsersAndSoftware := GetDetailQueries(&fleet.AppConfig{HostSettings: fleet.HostSettings{EnableHostUsers: true, EnableSoftwareInventory: true, EnableScheduledQueryStats: true}})
 	require.Len(t, queriesWithUsersAndSoftware, 16)
 	sortedKeysCompare(t, queriesWithUsersAndSoftware,
-		append(baseQueries, "users", "software_macos", "software_linux", "software_windows"))
+		append(baseQueries, "users", "software_macos", "software_linux", "software_windows", "scheduled_query_stats"))
 }
 
 func TestDirectIngestMDM(t *testing.T) {