diff --git a/changes/13489-implement-api-changes b/changes/13489-implement-api-changes new file mode 100644 index 0000000000..9b02ad5d73 --- /dev/null +++ b/changes/13489-implement-api-changes @@ -0,0 +1,2 @@ +* Add `GET /api/_version_/fleet/queries/{id}/report` API endpoint to retrieve the stored results of a given query. +* Add `discard_data` field to API query endpoints. diff --git a/cmd/fleetctl/get.go b/cmd/fleetctl/get.go index 3136dfd5b6..60b87335cb 100644 --- a/cmd/fleetctl/get.go +++ b/cmd/fleetctl/get.go @@ -14,14 +14,13 @@ import ( "github.com/fatih/color" "github.com/fleetdm/fleet/v4/pkg/secure" - kithttp "github.com/go-kit/kit/transport/http" - "gopkg.in/guregu/null.v3" - "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/service" "github.com/ghodss/yaml" + kithttp "github.com/go-kit/kit/transport/http" "github.com/olekukonko/tablewriter" "github.com/urfave/cli/v2" + "gopkg.in/guregu/null.v3" ) const ( @@ -451,6 +450,7 @@ func getQueriesCommand() *cli.Command { MinOsqueryVersion: query.MinOsqueryVersion, AutomationsEnabled: query.AutomationsEnabled, Logging: query.Logging, + DiscardData: query.DiscardData, }); err != nil { return fmt.Errorf("unable to print query: %w", err) } diff --git a/cmd/fleetctl/get_test.go b/cmd/fleetctl/get_test.go index a57d06413b..ff117e6420 100644 --- a/cmd/fleetctl/get_test.go +++ b/cmd/fleetctl/get_test.go @@ -1129,6 +1129,7 @@ kind: query spec: automations_enabled: false description: some desc + discard_data: false interval: 0 logging: "" min_osquery_version: "" @@ -1143,6 +1144,7 @@ kind: query spec: automations_enabled: false description: some desc 2 + discard_data: false interval: 0 logging: "" min_osquery_version: "" @@ -1157,6 +1159,7 @@ kind: query spec: automations_enabled: true description: some desc 4 + discard_data: false interval: 60 logging: differential_ignore_removals min_osquery_version: 5.3.0 @@ -1166,9 +1169,9 @@ spec: query: select 4; team: "" ` - expectedJSONGlobal := `{"kind":"query","apiVersion":"v1","spec":{"name":"query1","description":"some desc","query":"select 1;","team":"","interval":0,"observer_can_run":false,"platform":"","min_osquery_version":"","automations_enabled":false,"logging":""}} -{"kind":"query","apiVersion":"v1","spec":{"name":"query2","description":"some desc 2","query":"select 2;","team":"","interval":0,"observer_can_run":false,"platform":"","min_osquery_version":"","automations_enabled":false,"logging":""}} -{"kind":"query","apiVersion":"v1","spec":{"name":"query4","description":"some desc 4","query":"select 4;","team":"","interval":60,"observer_can_run":true,"platform":"darwin,windows","min_osquery_version":"5.3.0","automations_enabled":true,"logging":"differential_ignore_removals"}} + expectedJSONGlobal := `{"kind":"query","apiVersion":"v1","spec":{"name":"query1","description":"some desc","query":"select 1;","team":"","interval":0,"observer_can_run":false,"platform":"","min_osquery_version":"","automations_enabled":false,"logging":"","discard_data":false}} +{"kind":"query","apiVersion":"v1","spec":{"name":"query2","description":"some desc 2","query":"select 2;","team":"","interval":0,"observer_can_run":false,"platform":"","min_osquery_version":"","automations_enabled":false,"logging":"","discard_data":false}} +{"kind":"query","apiVersion":"v1","spec":{"name":"query4","description":"some desc 4","query":"select 4;","team":"","interval":60,"observer_can_run":true,"platform":"darwin,windows","min_osquery_version":"5.3.0","automations_enabled":true,"logging":"differential_ignore_removals","discard_data":false}} ` expectedTeam := `+--------+-------------+-----------+--------+----------------------------+ @@ -1192,6 +1195,7 @@ kind: query spec: automations_enabled: false description: some desc 3 + discard_data: false interval: 3600 logging: snapshot min_osquery_version: 5.4.0 @@ -1201,7 +1205,7 @@ spec: query: select 3; team: Foobar ` - expectedJSONTeam := `{"kind":"query","apiVersion":"v1","spec":{"name":"query3","description":"some desc 3","query":"select 3;","team":"Foobar","interval":3600,"observer_can_run":true,"platform":"darwin","min_osquery_version":"5.4.0","automations_enabled":false,"logging":"snapshot"}} + expectedJSONTeam := `{"kind":"query","apiVersion":"v1","spec":{"name":"query3","description":"some desc 3","query":"select 3;","team":"Foobar","interval":3600,"observer_can_run":true,"platform":"darwin","min_osquery_version":"5.4.0","automations_enabled":false,"logging":"snapshot","discard_data":false}} ` assert.Equal(t, expectedGlobal, runAppForTest(t, []string{"get", "queries"})) @@ -1277,6 +1281,7 @@ kind: query spec: automations_enabled: false description: some desc + discard_data: false interval: 0 logging: "" min_osquery_version: "" @@ -1286,7 +1291,7 @@ spec: query: select 1; team: "" ` - expectedJson := `{"kind":"query","apiVersion":"v1","spec":{"name":"globalQuery1","description":"some desc","query":"select 1;","team":"","interval":0,"observer_can_run":false,"platform":"","min_osquery_version":"","automations_enabled":false,"logging":""}} + expectedJson := `{"kind":"query","apiVersion":"v1","spec":{"name":"globalQuery1","description":"some desc","query":"select 1;","team":"","interval":0,"observer_can_run":false,"platform":"","min_osquery_version":"","automations_enabled":false,"logging":"","discard_data":false}} ` assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "query", "globalQuery1"})) @@ -1299,6 +1304,7 @@ kind: query spec: automations_enabled: true description: some team desc + discard_data: false interval: 3600 logging: differential min_osquery_version: 5.2.0 @@ -1308,7 +1314,7 @@ spec: query: select 2; team: Foobar ` - expectedJson = `{"kind":"query","apiVersion":"v1","spec":{"name":"teamQuery1","description":"some team desc","query":"select 2;","team":"Foobar","interval":3600,"observer_can_run":true,"platform":"linux","min_osquery_version":"5.2.0","automations_enabled":true,"logging":"differential"}} + expectedJson = `{"kind":"query","apiVersion":"v1","spec":{"name":"teamQuery1","description":"some team desc","query":"select 2;","team":"Foobar","interval":3600,"observer_can_run":true,"platform":"linux","min_osquery_version":"5.2.0","automations_enabled":true,"logging":"differential","discard_data":false}} ` assert.Equal(t, expectedYaml, runAppForTest(t, []string{"get", "query", "--team", "1", "teamQuery1"})) @@ -1433,6 +1439,7 @@ kind: query spec: automations_enabled: false description: some desc 2 + discard_data: false interval: 0 logging: "" min_osquery_version: "" @@ -1442,7 +1449,7 @@ spec: query: select 2; team: "" ` - expectedJson := `{"kind":"query","apiVersion":"v1","spec":{"name":"query2","description":"some desc 2","query":"select 2;","team":"","interval":0,"observer_can_run":true,"platform":"","min_osquery_version":"","automations_enabled":false,"logging":""}} + expectedJson := `{"kind":"query","apiVersion":"v1","spec":{"name":"query2","description":"some desc 2","query":"select 2;","team":"","interval":0,"observer_can_run":true,"platform":"","min_osquery_version":"","automations_enabled":false,"logging":"","discard_data":false}} ` assert.Equal(t, expected, runAppForTest(t, []string{"get", "queries"})) @@ -1510,6 +1517,7 @@ kind: query spec: automations_enabled: false description: some desc + discard_data: false interval: 0 logging: "" min_osquery_version: "" @@ -1524,6 +1532,7 @@ kind: query spec: automations_enabled: false description: some desc 2 + discard_data: false interval: 0 logging: "" min_osquery_version: "" @@ -1538,6 +1547,7 @@ kind: query spec: automations_enabled: false description: some desc 3 + discard_data: false interval: 0 logging: "" min_osquery_version: "" @@ -1547,9 +1557,9 @@ spec: query: select 3; team: "" ` - expectedJson := `{"kind":"query","apiVersion":"v1","spec":{"name":"query1","description":"some desc","query":"select 1;","team":"","interval":0,"observer_can_run":false,"platform":"","min_osquery_version":"","automations_enabled":false,"logging":""}} -{"kind":"query","apiVersion":"v1","spec":{"name":"query2","description":"some desc 2","query":"select 2;","team":"","interval":0,"observer_can_run":true,"platform":"","min_osquery_version":"","automations_enabled":false,"logging":""}} -{"kind":"query","apiVersion":"v1","spec":{"name":"query3","description":"some desc 3","query":"select 3;","team":"","interval":0,"observer_can_run":false,"platform":"","min_osquery_version":"","automations_enabled":false,"logging":""}} + expectedJson := `{"kind":"query","apiVersion":"v1","spec":{"name":"query1","description":"some desc","query":"select 1;","team":"","interval":0,"observer_can_run":false,"platform":"","min_osquery_version":"","automations_enabled":false,"logging":"","discard_data":false}} +{"kind":"query","apiVersion":"v1","spec":{"name":"query2","description":"some desc 2","query":"select 2;","team":"","interval":0,"observer_can_run":true,"platform":"","min_osquery_version":"","automations_enabled":false,"logging":"","discard_data":false}} +{"kind":"query","apiVersion":"v1","spec":{"name":"query3","description":"some desc 3","query":"select 3;","team":"","interval":0,"observer_can_run":false,"platform":"","min_osquery_version":"","automations_enabled":false,"logging":"","discard_data":false}} ` assert.Equal(t, expected, runAppForTest(t, []string{"get", "queries"})) diff --git a/cmd/fleetctl/testdata/convert_output.yml b/cmd/fleetctl/testdata/convert_output.yml index b09a8a0847..b2181e2763 100644 --- a/cmd/fleetctl/testdata/convert_output.yml +++ b/cmd/fleetctl/testdata/convert_output.yml @@ -4,6 +4,7 @@ kind: query spec: automations_enabled: false description: Retrieves the list of application scheme/protocol-based IPC handlers. + discard_data: false interval: 86400 logging: "" min_osquery_version: 1.4.7 @@ -18,6 +19,7 @@ kind: query spec: automations_enabled: false description: Retrieves the current disk encryption status for the target system. + discard_data: false interval: 86400 logging: "" min_osquery_version: 1.4.5 @@ -32,6 +34,7 @@ kind: query spec: automations_enabled: false description: Retrieves the current disk encryption status for the target system. + discard_data: false interval: 300 logging: "" min_osquery_version: 1.4.5 @@ -46,6 +49,7 @@ kind: query spec: automations_enabled: false description: Retrieve basic information about the physical disks of a system. + discard_data: false interval: 86400 logging: "" min_osquery_version: 1.4.7 @@ -60,6 +64,7 @@ kind: query spec: automations_enabled: false description: Retrieves the current filters and chains per filter in the target system. + discard_data: false interval: 3600 logging: "" min_osquery_version: 1.4.5 @@ -76,6 +81,7 @@ spec: description: Retrieves all the daemons that will run in the start of the target OSX system. + discard_data: false interval: 3600 logging: "" min_osquery_version: 1.4.5 @@ -90,6 +96,7 @@ kind: query spec: automations_enabled: false description: Retrieves the list of listening ports. + discard_data: false interval: 3600 logging: "" min_osquery_version: 1.4.7 @@ -104,6 +111,7 @@ kind: query spec: automations_enabled: false description: Retrieves the list of listening ports. + discard_data: false interval: 3600 logging: "" min_osquery_version: 1.4.7 @@ -118,6 +126,7 @@ kind: query spec: automations_enabled: false description: Lists the application bundle that owns a sandbox label. + discard_data: false interval: 86400 logging: "" min_osquery_version: 1.4.7 @@ -132,6 +141,7 @@ kind: query spec: automations_enabled: false description: System resource usage limits. + discard_data: false interval: 300 logging: "" min_osquery_version: 1.4.7 @@ -146,6 +156,7 @@ kind: query spec: automations_enabled: false description: System uptime. + discard_data: false interval: 600 logging: "" min_osquery_version: 1.4.7 @@ -160,6 +171,7 @@ kind: query spec: automations_enabled: false description: System uptime. + discard_data: false interval: 600 logging: "" min_osquery_version: 1.4.7 @@ -174,6 +186,7 @@ kind: query spec: automations_enabled: false description: System uptime. + discard_data: false interval: 600 logging: "" min_osquery_version: 1.4.7 @@ -188,6 +201,7 @@ kind: query spec: automations_enabled: false description: System uptime. + discard_data: false interval: 600 logging: "" min_osquery_version: 1.4.7 @@ -202,6 +216,7 @@ kind: query spec: automations_enabled: false description: Lists the application bundle that owns a sandbox label. + discard_data: false interval: 86400 logging: "" min_osquery_version: 1.4.7 @@ -216,6 +231,7 @@ kind: query spec: automations_enabled: false description: List of all user groups. + discard_data: false interval: 3600 logging: "" min_osquery_version: 1.4.7 @@ -230,6 +246,7 @@ kind: query spec: automations_enabled: false description: List of all user groups. + discard_data: false interval: 3600 logging: "" min_osquery_version: 1.4.7 @@ -244,6 +261,7 @@ kind: query spec: automations_enabled: false description: List of all user groups. + discard_data: false interval: 3600 logging: "" min_osquery_version: 1.4.7 @@ -258,6 +276,7 @@ kind: query spec: automations_enabled: false description: List of all user groups. + discard_data: false interval: 3600 logging: "" min_osquery_version: "" @@ -272,6 +291,7 @@ kind: query spec: automations_enabled: false description: List of all user groups. + discard_data: false interval: 3600 logging: "" min_osquery_version: 1.4.7 @@ -286,6 +306,7 @@ kind: query spec: automations_enabled: false description: List of all user groups. + discard_data: false interval: 3600 logging: "" min_osquery_version: 1.4.7 @@ -300,6 +321,7 @@ kind: query spec: automations_enabled: false description: List of all user groups. + discard_data: false interval: 3600 logging: "" min_osquery_version: "" @@ -314,6 +336,7 @@ kind: query spec: automations_enabled: false description: List of all user groups. + discard_data: false interval: 3600 logging: "" min_osquery_version: 1.4.7 @@ -328,6 +351,7 @@ kind: query spec: automations_enabled: false description: Extracted information from Windows crash logs (Minidumps). + discard_data: false interval: 3600 logging: "" min_osquery_version: 1.4.7 @@ -344,6 +368,7 @@ spec: description: Triggers one-off YARA query for files at the specified path. Requires one of sig_group, sigfile, or sigrule. + discard_data: false interval: 0 logging: "" min_osquery_version: 1.4.7 diff --git a/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json b/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json index e6ae712e28..2bca9327cb 100644 --- a/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json +++ b/cmd/fleetctl/testdata/expectedGetConfigAppConfigJson.json @@ -11,6 +11,7 @@ "server_settings": { "server_url": "", "live_query_disabled": false, + "query_reports_disabled": false, "enable_analytics": false, "deferred_save_host": false }, @@ -112,4 +113,4 @@ } } } -} +} \ No newline at end of file diff --git a/cmd/fleetctl/testdata/expectedGetConfigAppConfigYaml.yml b/cmd/fleetctl/testdata/expectedGetConfigAppConfigYaml.yml index 1c0d778685..0206e1cf76 100644 --- a/cmd/fleetctl/testdata/expectedGetConfigAppConfigYaml.yml +++ b/cmd/fleetctl/testdata/expectedGetConfigAppConfigYaml.yml @@ -48,6 +48,7 @@ spec: deferred_save_host: false enable_analytics: false live_query_disabled: false + query_reports_disabled: false server_url: "" smtp_settings: authentication_method: "" diff --git a/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json b/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json index 2030db5afe..632d37cc8f 100644 --- a/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json +++ b/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigJson.json @@ -11,6 +11,7 @@ "server_settings": { "server_url": "", "live_query_disabled": false, + "query_reports_disabled": false, "enable_analytics": false, "deferred_save_host": false }, @@ -174,4 +175,4 @@ } } } -} +} \ No newline at end of file diff --git a/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigYaml.yml b/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigYaml.yml index 9d3bf00ace..4c835b47e8 100644 --- a/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigYaml.yml +++ b/cmd/fleetctl/testdata/expectedGetConfigIncludeServerConfigYaml.yml @@ -87,6 +87,7 @@ spec: deferred_save_host: false enable_analytics: false live_query_disabled: false + query_reports_disabled: false server_url: "" smtp_settings: authentication_method: "" diff --git a/cmd/fleetctl/testdata/macosSetupExpectedAppConfigEmpty.yml b/cmd/fleetctl/testdata/macosSetupExpectedAppConfigEmpty.yml index 4fc311a8dd..bcd27f522e 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedAppConfigEmpty.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedAppConfigEmpty.yml @@ -48,6 +48,7 @@ spec: deferred_save_host: false enable_analytics: false live_query_disabled: false + query_reports_disabled: false server_url: https://example.org smtp_settings: authentication_method: "" diff --git a/cmd/fleetctl/testdata/macosSetupExpectedAppConfigSet.yml b/cmd/fleetctl/testdata/macosSetupExpectedAppConfigSet.yml index 72b5d2c599..0f94a3c0b5 100644 --- a/cmd/fleetctl/testdata/macosSetupExpectedAppConfigSet.yml +++ b/cmd/fleetctl/testdata/macosSetupExpectedAppConfigSet.yml @@ -48,6 +48,7 @@ spec: deferred_save_host: false enable_analytics: false live_query_disabled: false + query_reports_disabled: false server_url: https://example.org smtp_settings: authentication_method: "" diff --git a/cmd/fleetctl/upgrade_packs_test.go b/cmd/fleetctl/upgrade_packs_test.go index 1c903737d7..c30ae6e2c3 100644 --- a/cmd/fleetctl/upgrade_packs_test.go +++ b/cmd/fleetctl/upgrade_packs_test.go @@ -255,7 +255,7 @@ func TestFleetctlUpgradePacks_EmptyPacks(t *testing.T) { outputFile := filepath.Join(tempDir, "output.yml") // write some dummy data in the file, it should be overwritten - err := os.WriteFile(outputFile, []byte("dummy"), 0644) + err := os.WriteFile(outputFile, []byte("dummy"), 0o644) require.NoError(t, err) got := runAppForTest(t, []string{"upgrade-packs", "-o", outputFile}) @@ -328,6 +328,7 @@ kind: query spec: automations_enabled: false description: (converted from pack "p1", query "q1") + discard_data: false interval: 0 logging: snapshot min_osquery_version: "" @@ -342,6 +343,7 @@ kind: query spec: automations_enabled: true description: (converted from pack "p2", query "q2") + discard_data: false interval: 90 logging: differential min_osquery_version: "" @@ -356,6 +358,7 @@ kind: query spec: automations_enabled: true description: (converted from pack "p2", query "q2") + discard_data: false interval: 90 logging: differential min_osquery_version: "" @@ -371,7 +374,7 @@ spec: outputFile := filepath.Join(tempDir, "output.yml") // write some dummy data in the file, it should be overwritten - err := os.WriteFile(outputFile, []byte("dummy"), 0644) + err := os.WriteFile(outputFile, []byte("dummy"), 0o644) require.NoError(t, err) testUpgradePacksTimestamp = time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) @@ -394,7 +397,7 @@ func TestFleetctlUpgradePacks_NotAdmin(t *testing.T) { outputFile := filepath.Join(tempDir, "output.yml") // write some dummy data in the file, it should NOT be overwritten - err := os.WriteFile(outputFile, []byte("dummy"), 0644) + err := os.WriteFile(outputFile, []byte("dummy"), 0o644) require.NoError(t, err) // first try without the required output file flag @@ -422,7 +425,7 @@ func TestFleetctlUpgradePacks_NoPack(t *testing.T) { outputFile := filepath.Join(tempDir, "output.yml") // write some dummy data in the file, it should NOT be overwritten - err := os.WriteFile(outputFile, []byte("dummy"), 0644) + err := os.WriteFile(outputFile, []byte("dummy"), 0o644) require.NoError(t, err) got := runAppForTest(t, []string{"upgrade-packs", "-o", outputFile}) diff --git a/server/datastore/mysql/queries.go b/server/datastore/mysql/queries.go index 60f5e87210..94b92497bd 100644 --- a/server/datastore/mysql/queries.go +++ b/server/datastore/mysql/queries.go @@ -43,8 +43,9 @@ func (ds *Datastore) ApplyQueries(ctx context.Context, authorID uint, queries [] min_osquery_version, schedule_interval, automations_enabled, - logging_type - ) VALUES ( ?, ?, ?, ?, true, ?, ?, ?, ?, ?, ?, ?, ? ) + logging_type, + discard_data + ) VALUES ( ?, ?, ?, ?, true, ?, ?, ?, ?, ?, ?, ?, ?, ? ) ON DUPLICATE KEY UPDATE name = VALUES(name), description = VALUES(description), @@ -58,7 +59,8 @@ func (ds *Datastore) ApplyQueries(ctx context.Context, authorID uint, queries [] min_osquery_version = VALUES(min_osquery_version), schedule_interval = VALUES(schedule_interval), automations_enabled = VALUES(automations_enabled), - logging_type = VALUES(logging_type) + logging_type = VALUES(logging_type), + discard_data = VALUES(discard_data) ` stmt, err := tx.PrepareContext(ctx, insertSql) if err != nil { @@ -95,6 +97,7 @@ func (ds *Datastore) ApplyQueries(ctx context.Context, authorID uint, queries [] q.Interval, q.AutomationsEnabled, q.Logging, + q.DiscardData, ) if err != nil { return ctxerr.Wrap(ctx, err, "exec ApplyQueries insert") @@ -136,6 +139,7 @@ func (ds *Datastore) QueryByName( min_osquery_version, automations_enabled, logging_type, + discard_data, created_at, updated_at FROM queries @@ -364,6 +368,7 @@ func (ds *Datastore) Query(ctx context.Context, id uint) (*fleet.Query, error) { q.min_osquery_version, q.automations_enabled, q.logging_type, + q.discard_data, q.created_at, q.updated_at, q.discard_data, @@ -414,6 +419,7 @@ func (ds *Datastore) ListQueries(ctx context.Context, opt fleet.ListQueryOptions q.min_osquery_version, q.automations_enabled, q.logging_type, + q.discard_data, q.created_at, q.updated_at, q.discard_data, diff --git a/server/datastore/mysql/query_results.go b/server/datastore/mysql/query_results.go new file mode 100644 index 0000000000..d4aa6bc0cc --- /dev/null +++ b/server/datastore/mysql/query_results.go @@ -0,0 +1,126 @@ +package mysql + +import ( + "context" + "strings" + + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/jmoiron/sqlx" +) + +// OverwriteQueryResultRows overwrites the query result rows for a given query and host +// in a single transaction, ensuring that the number of rows for the given query +// does not exceed the maximum allowed +func (ds *Datastore) OverwriteQueryResultRows(ctx context.Context, rows []*fleet.ScheduledQueryResultRow) (err error) { + if len(rows) == 0 { + return nil + } + + err = ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { + // Since we assume all rows have the same queryID, take it from the first row + queryID := rows[0].QueryID + hostID := rows[0].HostID + + // Count how many rows are already in the database for the given queryID + var countExisting int + countStmt := ` + SELECT COUNT(*) FROM query_results WHERE query_id = ? + ` + err = sqlx.GetContext(ctx, tx, &countExisting, countStmt, queryID) + if err != nil { + return ctxerr.Wrap(ctx, err, "counting existing query results") + } + + if countExisting == fleet.MaxQueryReportRows { + // do not delete any rows if we are already at the limit + return nil + } + + // Delete rows based on the specific queryID and hostID + deleteStmt := ` + DELETE FROM query_results WHERE host_id = ? AND query_id = ? + ` + result, err := tx.ExecContext(ctx, deleteStmt, hostID, queryID) + if err != nil { + return ctxerr.Wrap(ctx, err, "deleting query results for host") + } + + // Count how many rows we deleted + countDeleted, err := result.RowsAffected() + if err != nil { + return ctxerr.Wrap(ctx, err, "fetching deleted row count") + } + + // Calculate how many new rows can be added given the maximum limit + netRowsAfterDeletion := countExisting - int(countDeleted) + allowedNewRows := fleet.MaxQueryReportRows - netRowsAfterDeletion + if allowedNewRows == 0 { + return nil + } + + if len(rows) > allowedNewRows { + rows = rows[:allowedNewRows] + } + + // Insert the new rows + valueStrings := make([]string, 0, len(rows)) + valueArgs := make([]interface{}, 0, len(rows)*4) + for _, row := range rows { + valueStrings = append(valueStrings, "(?, ?, ?, ?)") + valueArgs = append(valueArgs, queryID, hostID, row.LastFetched, row.Data) + } + + //nolint:gosec // SQL query is constructed using constant strings + insertStmt := ` + INSERT INTO query_results (query_id, host_id, last_fetched, data) VALUES + ` + strings.Join(valueStrings, ",") + + _, err = tx.ExecContext(ctx, insertStmt, valueArgs...) + if err != nil { + return ctxerr.Wrap(ctx, err, "inserting new rows") + } + + return nil + }) + + return ctxerr.Wrap(ctx, err, "overwriting query result rows") +} + +// TODO(lucas): Any chance we can store hostname in the query_results table? +// (to avoid having to left join hosts). +func (ds *Datastore) QueryResultRows(ctx context.Context, queryID uint) ([]*fleet.ScheduledQueryResultRow, error) { + selectStmt := ` + SELECT qr.query_id, qr.host_id, h.hostname, qr.last_fetched, qr.data + FROM query_results qr + LEFT JOIN hosts h ON (qr.host_id=h.id) + WHERE query_id = ? + ` + results := []*fleet.ScheduledQueryResultRow{} + err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, selectStmt, queryID) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "selecting query result rows") + } + + return results, nil +} + +func (ds *Datastore) ResultCountForQuery(ctx context.Context, queryID uint) (int, error) { + var count int + err := sqlx.GetContext(ctx, ds.reader(ctx), &count, `select count(*) from query_results where query_id = ?`, queryID) + if err != nil { + return 0, ctxerr.Wrap(ctx, err, "counting query results for query") + } + + return count, nil +} + +func (ds *Datastore) ResultCountForQueryAndHost(ctx context.Context, queryID, hostID uint) (int, error) { + var count int + err := sqlx.GetContext(ctx, ds.reader(ctx), &count, `select count(*) from query_results where query_id = ? AND host_id = ?`, queryID, hostID) + if err != nil { + return 0, ctxerr.Wrap(ctx, err, "counting query results for query and host") + } + + return count, nil +} diff --git a/server/datastore/mysql/query_results_test.go b/server/datastore/mysql/query_results_test.go new file mode 100644 index 0000000000..2e259ce924 --- /dev/null +++ b/server/datastore/mysql/query_results_test.go @@ -0,0 +1,402 @@ +package mysql + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/fleetdm/fleet/v4/server/test" + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/require" +) + +func TestQueryResults(t *testing.T) { + ds := CreateMySQLDS(t) + + cases := []struct { + name string + fn func(t *testing.T, ds *Datastore) + }{ + {"Save", saveQueryResultRows}, + {"Get", getQueryResultRows}, + {"CountForQuery", testCountResultsForQuery}, + {"CountForQueryAndHost", testCountResultsForQueryAndHost}, + {"Overwrite", testOverwriteQueryResultRows}, + {"MaxRows", testQueryResultRowsDoNotExceedMaxRows}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + defer TruncateTables(t, ds) + c.fn(t, ds) + }) + } +} + +func saveQueryResultRows(t *testing.T, ds *Datastore) { + user := test.NewUser(t, ds, "Test User", "test@example.com", true) + query := test.NewQuery(t, ds, nil, "New Query", "SELECT 1", user.ID, true) + host := test.NewHost(t, ds, "hostname123", "192.168.1.100", "1234", "UI8XB1223", time.Now()) + + mockTime := time.Now().UTC().Truncate(time.Second) + + resultRows := []*fleet.ScheduledQueryResultRow{ + { + QueryID: query.ID, + HostID: host.ID, + LastFetched: mockTime, + Data: json.RawMessage( + `{"model": "USB Keyboard", "vendor": "Apple Inc."}`, + ), + }, + { + QueryID: query.ID, + HostID: host.ID, + LastFetched: mockTime, + Data: json.RawMessage( + `{"model": "USB Mouse", "vendor": "Logitech"}`, + ), + }, + } + + err := ds.SaveQueryResultRows(context.Background(), resultRows) + require.NoError(t, err) +} + +func getQueryResultRows(t *testing.T, ds *Datastore) { + user := test.NewUser(t, ds, "Test User", "test@example.com", true) + query := test.NewQuery(t, ds, nil, "New Query", "SELECT 1", user.ID, true) + host := test.NewHost(t, ds, "hostname123", "192.168.1.100", "1234", "UI8XB1223", time.Now()) + + mockTime := time.Now().UTC().Truncate(time.Second) + + // Insert 2 Result Rows for Query1 + resultRows := []*fleet.ScheduledQueryResultRow{ + { + QueryID: query.ID, + HostID: host.ID, + LastFetched: mockTime, + Data: json.RawMessage( + `{"model": "USB Keyboard", "vendor": "Apple Inc."}`, + ), + }, + { + QueryID: query.ID, + HostID: host.ID, + LastFetched: mockTime, + Data: json.RawMessage( + `{"model": "USB Mouse", "vendor": "Logitech"}`, + ), + }, + } + + err := ds.SaveQueryResultRows(context.Background(), resultRows) + require.NoError(t, err) + + // Insert Result Row for different Scheduled Query + query2 := test.NewQuery(t, ds, nil, "New Query 2", "SELECT 1", user.ID, true) + resultRow3 := []*fleet.ScheduledQueryResultRow{ + { + QueryID: query2.ID, + HostID: host.ID, + LastFetched: mockTime, + Data: json.RawMessage( + `{"model": "USB Hub","vendor": "Logitech"}`, + ), + }, + } + + err = ds.SaveQueryResultRows(context.Background(), resultRow3) + require.NoError(t, err) + + // Assert that Query1 returns 2 results + results, err := ds.QueryResultRowsForHost(context.Background(), resultRows[0].QueryID, resultRows[0].HostID) + require.NoError(t, err) + require.Len(t, results, 2) + require.Equal(t, resultRows[0].QueryID, results[0].QueryID) + require.Equal(t, resultRows[0].HostID, results[0].HostID) + require.Equal(t, resultRows[0].LastFetched.Unix(), results[0].LastFetched.Unix()) + require.JSONEq(t, string(resultRows[0].Data), string(results[0].Data)) + require.Equal(t, resultRows[1].QueryID, results[1].QueryID) + require.Equal(t, resultRows[1].HostID, results[1].HostID) + require.Equal(t, resultRows[1].LastFetched.Unix(), results[1].LastFetched.Unix()) + require.JSONEq(t, string(resultRows[1].Data), string(results[1].Data)) + + // Assert that Query2 returns 1 result + results, err = ds.QueryResultRowsForHost(context.Background(), resultRow3[0].QueryID, resultRow3[0].HostID) + require.NoError(t, err) + require.Len(t, results, 1) + require.Equal(t, resultRow3[0].QueryID, results[0].QueryID) + require.Equal(t, resultRow3[0].HostID, results[0].HostID) + require.Equal(t, resultRow3[0].LastFetched.Unix(), results[0].LastFetched.Unix()) + require.JSONEq(t, string(resultRow3[0].Data), string(results[0].Data)) + + // Assert that QueryResultRows returns empty slice when no results are found + results, err = ds.QueryResultRowsForHost(context.Background(), 999, 999) + require.NoError(t, err) + require.Len(t, results, 0) +} + +func testCountResultsForQuery(t *testing.T, ds *Datastore) { + user := test.NewUser(t, ds, "Test User", "test@example.com", true) + query1 := test.NewQuery(t, ds, nil, "New Query", "SELECT 1", user.ID, true) + query2 := test.NewQuery(t, ds, nil, "New Query 2", "SELECT 1", user.ID, true) + host := test.NewHost(t, ds, "hostname123", "192.168.1.100", "1234", "UI8XB1223", time.Now()) + + mockTime := time.Now().UTC().Truncate(time.Second) + + // Insert 1 Result Row for Query1 + resultRow := []*fleet.ScheduledQueryResultRow{ + { + QueryID: query1.ID, + HostID: host.ID, + LastFetched: mockTime, + Data: json.RawMessage(`{ + "model": "USB Keyboard", + "vendor": "Apple Inc." + }`), + }, + } + + err := ds.SaveQueryResultRows(context.Background(), resultRow) + require.NoError(t, err) + + // Insert 5 Result Rows for Query2 + resultRow2 := []*fleet.ScheduledQueryResultRow{ + { + QueryID: query2.ID, + HostID: host.ID, + LastFetched: mockTime, + Data: json.RawMessage(`{ + "model": "USB Mouse", + "vendor": "Apple Inc." + }`), + }, + } + for i := 0; i < 5; i++ { + err = ds.SaveQueryResultRows(context.Background(), resultRow2) + require.NoError(t, err) + } + + // Assert that ResultCountForQuery returns 1 + count, err := ds.ResultCountForQuery(context.Background(), query1.ID) + require.NoError(t, err) + require.Equal(t, 1, count) + + // Assert that ResultCountForQuery returns 5 + count, err = ds.ResultCountForQuery(context.Background(), query2.ID) + require.NoError(t, err) + require.Equal(t, 5, count) + + // Returns empty result when no results are found + count, err = ds.ResultCountForQuery(context.Background(), 999) + require.NoError(t, err) + require.Equal(t, 0, count) +} + +func testCountResultsForQueryAndHost(t *testing.T, ds *Datastore) { + user := test.NewUser(t, ds, "Test User", "test@example.com", true) + query1 := test.NewQuery(t, ds, nil, "New Query", "SELECT 1", user.ID, true) + query2 := test.NewQuery(t, ds, nil, "New Query 2", "SELECT 1", user.ID, true) + host := test.NewHost(t, ds, "host1", "192.168.1.100", "1234", "UI8XB1223", time.Now()) + host2 := test.NewHost(t, ds, "host2", "192.168.1.101", "4567", "UI8XB1224", time.Now()) + + mockTime := time.Now().UTC().Truncate(time.Second) + + resultRows := []*fleet.ScheduledQueryResultRow{ + { + QueryID: query1.ID, + HostID: host.ID, + LastFetched: mockTime, + Data: json.RawMessage(`{ + "model": "USB Keyboard", + "vendor": "Apple Inc." + }`), + }, + { + QueryID: query1.ID, + HostID: host.ID, + LastFetched: mockTime, + Data: json.RawMessage(`{ + "model": "USB Mouse", + "vendor": "Logitech" + }`), + }, + { + QueryID: query1.ID, + HostID: host2.ID, + LastFetched: mockTime, + Data: json.RawMessage(`{ + "model": "USB Mouse", + "vendor": "Logitech" + }`), + }, + { + QueryID: query2.ID, + HostID: host.ID, + LastFetched: mockTime, + Data: json.RawMessage(`{ + "foo": "bar" + }`), + }, + } + + err := ds.SaveQueryResultRows(context.Background(), resultRows) + require.NoError(t, err) + + // Assert that Query1 returns 2 + count, err := ds.ResultCountForQueryAndHost(context.Background(), query1.ID, host.ID) + require.NoError(t, err) + require.Equal(t, 2, count) + + // Assert that ResultCountForQuery returns 1 + count, err = ds.ResultCountForQueryAndHost(context.Background(), query2.ID, host.ID) + require.NoError(t, err) + require.Equal(t, 1, count) + + // Returns empty result when no results are found + count, err = ds.ResultCountForQueryAndHost(context.Background(), 999, host.ID) + require.NoError(t, err) + require.Equal(t, 0, count) +} + +func testOverwriteQueryResultRows(t *testing.T, ds *Datastore) { + user := test.NewUser(t, ds, "Test User", "test@example.com", true) + query := test.NewQuery(t, ds, nil, "Overwrite Test Query", "SELECT 1", user.ID, true) + host := test.NewHost(t, ds, "hostname1234", "192.168.1.101", "12345", "UI8XB1224", time.Now()) + + mockTime := time.Now().UTC().Truncate(time.Second) + + // Insert initial Result Rows + initialRows := []*fleet.ScheduledQueryResultRow{ + { + QueryID: query.ID, + HostID: host.ID, + LastFetched: mockTime, + Data: json.RawMessage( + `{"model": "USB Keyboard", "vendor": "Apple Inc."}`, + ), + }, + } + + err := ds.SaveQueryResultRows(context.Background(), initialRows) + require.NoError(t, err) + + // Overwrite Result Rows with new data + newMockTime := mockTime.Add(2 * time.Minute) + overwriteRows := []*fleet.ScheduledQueryResultRow{ + { + QueryID: query.ID, + HostID: host.ID, + LastFetched: newMockTime, + Data: json.RawMessage( + `{"model": "USB Mouse", "vendor": "Logitech"}`, + ), + }, + } + + err = ds.OverwriteQueryResultRows(context.Background(), overwriteRows) + require.NoError(t, err) + + // Assert that we get the overwritten data (1 result with USB Mouse data) + results, err := ds.QueryResultRowsForHost(context.Background(), overwriteRows[0].QueryID, overwriteRows[0].HostID) + require.NoError(t, err) + require.Len(t, results, 1) + require.Equal(t, overwriteRows[0].QueryID, results[0].QueryID) + require.Equal(t, overwriteRows[0].HostID, results[0].HostID) + require.Equal(t, overwriteRows[0].LastFetched.Unix(), results[0].LastFetched.Unix()) + require.JSONEq(t, string(overwriteRows[0].Data), string(results[0].Data)) +} + +func testQueryResultRowsDoNotExceedMaxRows(t *testing.T, ds *Datastore) { + user := test.NewUser(t, ds, "Test User", "test@example.com", true) + query := test.NewQuery(t, ds, nil, "Overwrite Test Query", "SELECT 1", user.ID, true) + host := test.NewHost(t, ds, "hostname1", "192.168.1.101", "12345", "UI8XB1224", time.Now()) + + mockTime := time.Now().UTC().Truncate(time.Second) + + // Generate more than max rows + rows := fleet.MaxQueryReportRows + 50 + largeBatchRows := make([]*fleet.ScheduledQueryResultRow, rows) + for i := 0; i < rows; i++ { + largeBatchRows[i] = &fleet.ScheduledQueryResultRow{ + QueryID: query.ID, + HostID: host.ID, + LastFetched: mockTime, + Data: json.RawMessage(`{"model": "Bulk Mouse", "vendor": "BulkTech"}`), + } + } + + err := ds.OverwriteQueryResultRows(context.Background(), largeBatchRows) + require.NoError(t, err) + + // Confirm only max rows are stored for the queryID + allResults, err := ds.QueryResultRowsForHost(context.Background(), query.ID, host.ID) + require.NoError(t, err) + require.Len(t, allResults, fleet.MaxQueryReportRows) + + // Confirm that new rows are not added when the max is reached + host2 := test.NewHost(t, ds, "hostname2", "192.168.1.102", "678910", "UI8XB1225", time.Now()) + newMockTime := mockTime.Add(2 * time.Minute) + overwriteRows := []*fleet.ScheduledQueryResultRow{ + { + QueryID: query.ID, + HostID: host2.ID, + LastFetched: newMockTime, + Data: json.RawMessage( + `{"model": "USB Mouse", "vendor": "Logitech"}`, + ), + }, + } + + err = ds.OverwriteQueryResultRows(context.Background(), overwriteRows) + require.NoError(t, err) + + host2Results, err := ds.QueryResultRowsForHost(context.Background(), query.ID, host2.ID) + require.NoError(t, err) + require.Len(t, host2Results, 0) +} + +func (ds *Datastore) SaveQueryResultRows(ctx context.Context, rows []*fleet.ScheduledQueryResultRow) error { + if len(rows) == 0 { + return nil // Nothing to insert + } + + valueStrings := make([]string, 0, len(rows)) + valueArgs := make([]interface{}, 0, len(rows)*4) + + for _, row := range rows { + valueStrings = append(valueStrings, "(?, ?, ?, ?)") + valueArgs = append(valueArgs, row.QueryID, row.HostID, row.LastFetched, row.Data) + } + + insertStmt := fmt.Sprintf(` + INSERT INTO query_results (query_id, host_id, last_fetched, data) + VALUES %s + `, strings.Join(valueStrings, ",")) + + _, err := ds.writer(ctx).ExecContext(ctx, insertStmt, valueArgs...) + if err != nil { + return err + } + + return nil +} + +func (ds *Datastore) QueryResultRowsForHost(ctx context.Context, queryID, hostID uint) ([]*fleet.ScheduledQueryResultRow, error) { + selectStmt := ` + SELECT query_id, host_id, last_fetched, data FROM query_results + WHERE query_id = ? AND host_id = ? + ` + results := []*fleet.ScheduledQueryResultRow{} + err := sqlx.SelectContext(ctx, ds.reader(ctx), &results, selectStmt, queryID, hostID) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "selecting query result rows for host") + } + + return results, nil +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 22e0b6a6ff..de39bc7e49 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -40,7 +40,7 @@ CREATE TABLE `app_config_json` ( UNIQUE KEY `id` (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `app_config_json` VALUES (1,'{\"mdm\": {\"macos_setup\": {\"bootstrap_package\": null, \"macos_setup_assistant\": null, \"enable_end_user_authentication\": false}, \"macos_updates\": {\"deadline\": null, \"minimum_version\": null}, \"macos_settings\": {\"custom_settings\": null, \"enable_disk_encryption\": false}, \"macos_migration\": {\"mode\": \"\", \"enable\": false, \"webhook_url\": \"\"}, \"apple_bm_default_team\": \"\", \"apple_bm_terms_expired\": false, \"enabled_and_configured\": false, \"end_user_authentication\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"issuer_uri\": \"\", \"metadata_url\": \"\"}, \"windows_enabled_and_configured\": false, \"apple_bm_enabled_and_configured\": false}, \"features\": {\"enable_host_users\": true, \"enable_software_inventory\": false}, \"org_info\": {\"org_name\": \"\", \"contact_url\": \"\", \"org_logo_url\": \"\", \"org_logo_url_light_background\": \"\"}, \"integrations\": {\"jira\": null, \"zendesk\": null}, \"sso_settings\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"enable_sso\": false, \"issuer_uri\": \"\", \"metadata_url\": \"\", \"idp_image_url\": \"\", \"enable_jit_role_sync\": false, \"enable_sso_idp_login\": false, \"enable_jit_provisioning\": false}, \"agent_options\": {\"config\": {\"options\": {\"logger_plugin\": \"tls\", \"pack_delimiter\": \"/\", \"logger_tls_period\": 10, \"distributed_plugin\": \"tls\", \"disable_distributed\": false, \"logger_tls_endpoint\": \"/api/osquery/log\", \"distributed_interval\": 10, \"distributed_tls_max_attempts\": 3}, \"decorators\": {\"load\": [\"SELECT uuid AS host_uuid FROM system_info;\", \"SELECT hostname AS hostname FROM system_info;\"]}}, \"overrides\": {}}, \"fleet_desktop\": {\"transparency_url\": \"\"}, \"smtp_settings\": {\"port\": 587, \"domain\": \"\", \"server\": \"\", \"password\": \"\", \"user_name\": \"\", \"configured\": false, \"enable_smtp\": false, \"enable_ssl_tls\": true, \"sender_address\": \"\", \"enable_start_tls\": true, \"verify_ssl_certs\": true, \"authentication_type\": \"0\", \"authentication_method\": \"0\"}, \"server_settings\": {\"server_url\": \"\", \"enable_analytics\": false, \"deferred_save_host\": false, \"live_query_disabled\": false}, \"webhook_settings\": {\"interval\": \"0s\", \"host_status_webhook\": {\"days_count\": 0, \"destination_url\": \"\", \"host_percentage\": 0, \"enable_host_status_webhook\": false}, \"vulnerabilities_webhook\": {\"destination_url\": \"\", \"host_batch_size\": 0, \"enable_vulnerabilities_webhook\": false}, \"failing_policies_webhook\": {\"policy_ids\": null, \"destination_url\": \"\", \"host_batch_size\": 0, \"enable_failing_policies_webhook\": false}}, \"host_expiry_settings\": {\"host_expiry_window\": 0, \"host_expiry_enabled\": false}, \"vulnerability_settings\": {\"databases_path\": \"\"}}','2020-01-01 01:01:01','2020-01-01 01:01:01'); +INSERT INTO `app_config_json` VALUES (1,'{\"mdm\": {\"macos_setup\": {\"bootstrap_package\": null, \"macos_setup_assistant\": null, \"enable_end_user_authentication\": false}, \"macos_updates\": {\"deadline\": null, \"minimum_version\": null}, \"macos_settings\": {\"custom_settings\": null, \"enable_disk_encryption\": false}, \"macos_migration\": {\"mode\": \"\", \"enable\": false, \"webhook_url\": \"\"}, \"apple_bm_default_team\": \"\", \"apple_bm_terms_expired\": false, \"enabled_and_configured\": false, \"end_user_authentication\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"issuer_uri\": \"\", \"metadata_url\": \"\"}, \"windows_enabled_and_configured\": false, \"apple_bm_enabled_and_configured\": false}, \"features\": {\"enable_host_users\": true, \"enable_software_inventory\": false}, \"org_info\": {\"org_name\": \"\", \"contact_url\": \"\", \"org_logo_url\": \"\", \"org_logo_url_light_background\": \"\"}, \"integrations\": {\"jira\": null, \"zendesk\": null}, \"sso_settings\": {\"idp_name\": \"\", \"metadata\": \"\", \"entity_id\": \"\", \"enable_sso\": false, \"issuer_uri\": \"\", \"metadata_url\": \"\", \"idp_image_url\": \"\", \"enable_jit_role_sync\": false, \"enable_sso_idp_login\": false, \"enable_jit_provisioning\": false}, \"agent_options\": {\"config\": {\"options\": {\"logger_plugin\": \"tls\", \"pack_delimiter\": \"/\", \"logger_tls_period\": 10, \"distributed_plugin\": \"tls\", \"disable_distributed\": false, \"logger_tls_endpoint\": \"/api/osquery/log\", \"distributed_interval\": 10, \"distributed_tls_max_attempts\": 3}, \"decorators\": {\"load\": [\"SELECT uuid AS host_uuid FROM system_info;\", \"SELECT hostname AS hostname FROM system_info;\"]}}, \"overrides\": {}}, \"fleet_desktop\": {\"transparency_url\": \"\"}, \"smtp_settings\": {\"port\": 587, \"domain\": \"\", \"server\": \"\", \"password\": \"\", \"user_name\": \"\", \"configured\": false, \"enable_smtp\": false, \"enable_ssl_tls\": true, \"sender_address\": \"\", \"enable_start_tls\": true, \"verify_ssl_certs\": true, \"authentication_type\": \"0\", \"authentication_method\": \"0\"}, \"server_settings\": {\"server_url\": \"\", \"enable_analytics\": false, \"deferred_save_host\": false, \"live_query_disabled\": false, \"query_reports_disabled\": false}, \"webhook_settings\": {\"interval\": \"0s\", \"host_status_webhook\": {\"days_count\": 0, \"destination_url\": \"\", \"host_percentage\": 0, \"enable_host_status_webhook\": false}, \"vulnerabilities_webhook\": {\"destination_url\": \"\", \"host_batch_size\": 0, \"enable_vulnerabilities_webhook\": false}, \"failing_policies_webhook\": {\"policy_ids\": null, \"destination_url\": \"\", \"host_batch_size\": 0, \"enable_failing_policies_webhook\": false}}, \"host_expiry_settings\": {\"host_expiry_window\": 0, \"host_expiry_enabled\": false}, \"vulnerability_settings\": {\"databases_path\": \"\"}}','2020-01-01 01:01:01','2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `carve_blocks` ( diff --git a/server/fleet/app.go b/server/fleet/app.go index 6a77190739..2b9ce6503f 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -675,11 +675,12 @@ const DefaultOrgInfoContactURL = "https://fleetdm.com/company/contact" // ServerSettings contains general settings about the Fleet application. type ServerSettings struct { - ServerURL string `json:"server_url"` - LiveQueryDisabled bool `json:"live_query_disabled"` - EnableAnalytics bool `json:"enable_analytics"` - DebugHostIDs []uint `json:"debug_host_ids,omitempty"` - DeferredSaveHost bool `json:"deferred_save_host"` + ServerURL string `json:"server_url"` + LiveQueryDisabled bool `json:"live_query_disabled"` + EnableAnalytics bool `json:"enable_analytics"` + DebugHostIDs []uint `json:"debug_host_ids,omitempty"` + DeferredSaveHost bool `json:"deferred_save_host"` + QueryReportsDisabled bool `json:"query_reports_disabled"` } // HostExpirySettings contains settings pertaining to automatic host expiry. diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 0e4f2eacef..447fecdddc 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -391,6 +391,15 @@ type Datastore interface { // scheduled query did not exist. ScheduledQueryIDsByName(ctx context.Context, batchSize int, packAndSchedQueryNames ...[2]string) ([]uint, error) + /////////////////////////////////////////////////////////////////////////////// + // QueryResultsStore + + // QueryResultRows returns all the stored results of a query (from all hosts). + QueryResultRows(ctx context.Context, queryID uint) ([]*ScheduledQueryResultRow, error) + ResultCountForQuery(ctx context.Context, queryID uint) (int, error) + ResultCountForQueryAndHost(ctx context.Context, queryID, hostID uint) (int, error) + OverwriteQueryResultRows(ctx context.Context, rows []*ScheduledQueryResultRow) error + /////////////////////////////////////////////////////////////////////////////// // TeamStore diff --git a/server/fleet/osquery.go b/server/fleet/osquery.go index 737e4aed4d..8445977635 100644 --- a/server/fleet/osquery.go +++ b/server/fleet/osquery.go @@ -10,7 +10,8 @@ type OsqueryStatus int const ( // StatusOK is the success code returned by osquery - StatusOK OsqueryStatus = 0 + StatusOK OsqueryStatus = 0 + MaxQueryReportRows int = 1000 ) // QueryContent is the format of a query stanza in an osquery configuration. diff --git a/server/fleet/queries.go b/server/fleet/queries.go index 98ac351922..6c0b86d0fd 100644 --- a/server/fleet/queries.go +++ b/server/fleet/queries.go @@ -1,6 +1,7 @@ package fleet import ( + "encoding/json" "errors" "fmt" "strings" @@ -36,7 +37,10 @@ type QueryPayload struct { AutomationsEnabled *bool `json:"automations_enabled"` // Logging is set to "snapshot" if not set when creating a query. Logging *string `json:"logging"` - // DiscardData is set to false if not set when creating a query. + // DiscardData indicates if the scheduled query results should be discarded (true) + // or kept (false) in a query report. + // + // If not set during creation of a query, then the default value is false. DiscardData *bool `json:"discard_data"` } @@ -93,7 +97,8 @@ type Query struct { // // This field has null values if the query did not run as a schedule on any host. AggregatedStats `json:"stats"` - // DiscardData indicates if the scheduled query results should be discarded (true) or kept (false) in a query report. + // DiscardData indicates if the scheduled query results should be discarded (true) + // or kept (false) in a query report. DiscardData bool `json:"discard_data" db:"discard_data"` } @@ -290,6 +295,11 @@ type QuerySpec struct { AutomationsEnabled bool `json:"automations_enabled"` // Logging is set to "snapshot" if not set. Logging string `json:"logging"` + // DiscardData indicates if the scheduled query results should be discarded (true) + // or kept (false) in a query report. + // + // If not set, then the default value is false. + DiscardData bool `json:"discard_data"` } func LoadQueriesFromYaml(yml string) ([]*Query, error) { @@ -359,3 +369,64 @@ type QueryStats struct { UserTime int `json:"user_time" db:"user_time"` WallTime int `json:"wall_time" db:"wall_time"` } + +// MapQueryReportsResultsToRows converts the scheduled query results as stored in Fleet's database +// to HostQueryResultRows to be exposed to the API. +func MapQueryReportResultsToRows(rows []*ScheduledQueryResultRow) ([]HostQueryResultRow, error) { + var results []HostQueryResultRow + for _, row := range rows { + var columns map[string]string + if err := json.Unmarshal(row.Data, &columns); err != nil { + return nil, err + } + results = append(results, HostQueryResultRow{ + HostID: row.HostID, + Hostname: row.Hostname, + LastFetched: row.LastFetched, + Columns: columns, + }) + } + return results, nil +} + +// HostQueryResultRow contains a single scheduled query result row from a host. +// This type is used to expose the results on the API. +type HostQueryResultRow struct { + // HostID is the unique ID of the host. + HostID uint `json:"host_id"` + // Hostname is the host's hostname. + Hostname string `json:"host_name"` + // LastFetched is the time this result row was received. + LastFetched time.Time `json:"last_fetched"` + // Columns contains the key-value pairs of a result row. + // The map key is the name of the column, and the map value is the value. + Columns map[string]string `json:"columns"` +} + +// ScheduledQueryResult holds results of a scheduled query received from a osquery agent. +type ScheduledQueryResult struct { + // QueryName is the name of the query. + QueryName string `json:"name,omitempty"` + // OsqueryHostID is the identifier of the host. + OsqueryHostID string `json:"hostIdentifier"` + // Snapshot holds the result rows. It's an array of maps, where the map keys + // are column names and map values are the values. + Snapshot []json.RawMessage `json:"snapshot"` + // LastFetched is the time this result was received. + UnixTime uint `json:"unixTime"` +} + +// ScheduledQueryResultRow is a scheduled query result row. +type ScheduledQueryResultRow struct { + // QueryID is the unique identifier of the query. + QueryID uint `db:"query_id"` + // HostID is the unique identifier of the host. + HostID uint `db:"host_id"` + // Hostname is the host's hostname. + Hostname string `db:"hostname"` + // Data holds a single result row. It holds a map where the map keys + // are column names and map values are the values. + Data json.RawMessage `db:"data"` + // LastFetched is the time this result was received. + LastFetched time.Time `db:"last_fetched"` +} diff --git a/server/fleet/queries_test.go b/server/fleet/queries_test.go index ca0341d348..32eb9e10c2 100644 --- a/server/fleet/queries_test.go +++ b/server/fleet/queries_test.go @@ -1,7 +1,9 @@ package fleet import ( + "encoding/json" "testing" + "time" "github.com/fleetdm/fleet/v4/server/ptr" "github.com/stretchr/testify/assert" @@ -210,3 +212,216 @@ func TestVerifyQueryPlatforms(t *testing.T) { }) } } + +func TestMapQueryReportResultRows(t *testing.T) { + macOSUSBDevicesLastFetched := time.Now() + ubuntuUSBDevicesLastFetched := time.Now().Add(-1 * time.Hour) + macOSOsqueryInfoLastFetched := time.Now().Add(-2 * time.Hour) + for _, tc := range []struct { + name string + rows []*ScheduledQueryResultRow + expected []HostQueryResultRow + shouldFail bool + }{ + { + name: "USB devices query with results from a macOS and Linux host", + rows: []*ScheduledQueryResultRow{ + { + HostID: 1, + Hostname: "macOS host", + LastFetched: macOSUSBDevicesLastFetched, + Data: json.RawMessage(`{ + "class": "9", + "model": "AppleUSBVHCIBCE Root Hub Simulation", + "model_id": "8000", + "protocol": "", + "removable": "0", + "serial": "0", + "subclass": "255", + "usb_address": "", + "usb_port": "", + "vendor": "Apple Inc.", + "vendor_id": "05bc", + "version": "0.0" + }`), + }, + { + HostID: 1, + Hostname: "macOS host", + LastFetched: macOSUSBDevicesLastFetched, + Data: json.RawMessage(`{ + "class": "9", + "model": "AppleUSBXHCI Root Hub Simulation", + "model_id": "8007", + "protocol": "", + "removable": "0", + "serial": "0", + "subclass": "255", + "usb_address": "", + "usb_port": "", + "vendor": "Apple Inc.", + "vendor_id": "05ac", + "version": "0.0" + }`), + }, + { + HostID: 2, + Hostname: "ubuntu host", + LastFetched: ubuntuUSBDevicesLastFetched, + Data: json.RawMessage(`{ + "class": "9", + "model": "1.1 root hub", + "model_id": "0001", + "protocol": "0", + "removable": "-1", + "serial": "0000:02:00.0", + "subclass": "0", + "usb_address": "1", + "usb_port": "1", + "vendor": "Linux Foundation", + "vendor_id": "1d6b", + "version": "0602" + }`), + }, + }, + expected: []HostQueryResultRow{ + { + HostID: 1, + Hostname: "macOS host", + LastFetched: macOSUSBDevicesLastFetched, + Columns: map[string]string{ + "class": "9", + "model": "AppleUSBVHCIBCE Root Hub Simulation", + "model_id": "8000", + "protocol": "", + "removable": "0", + "serial": "0", + "subclass": "255", + "usb_address": "", + "usb_port": "", + "vendor": "Apple Inc.", + "vendor_id": "05bc", + "version": "0.0", + }, + }, + { + HostID: 1, + Hostname: "macOS host", + LastFetched: macOSUSBDevicesLastFetched, + Columns: map[string]string{ + "class": "9", + "model": "AppleUSBXHCI Root Hub Simulation", + "model_id": "8007", + "protocol": "", + "removable": "0", + "serial": "0", + "subclass": "255", + "usb_address": "", + "usb_port": "", + "vendor": "Apple Inc.", + "vendor_id": "05ac", + "version": "0.0", + }, + }, + { + HostID: 2, + Hostname: "ubuntu host", + LastFetched: ubuntuUSBDevicesLastFetched, + Columns: map[string]string{ + "class": "9", + "model": "1.1 root hub", + "model_id": "0001", + "protocol": "0", + "removable": "-1", + "serial": "0000:02:00.0", + "subclass": "0", + "usb_address": "1", + "usb_port": "1", + "vendor": "Linux Foundation", + "vendor_id": "1d6b", + "version": "0602", + }, + }, + }, + shouldFail: false, + }, + { + name: "macOS osquery_info result", + rows: []*ScheduledQueryResultRow{ + { + HostID: 1, + Hostname: "macOS host", + LastFetched: macOSOsqueryInfoLastFetched, + Data: json.RawMessage(`{ + "build_distro": "10.14", + "build_platform": "darwin", + "config_hash": "eed0d8296e5f90b790a23814a9db7a127b13498d", + "config_valid": "1", + "extensions": "active", + "instance_id": "7f02ff0f-f8a7-4ba9-a1d2-66836b154f4a", + "pid": "96730", + "platform_mask": "21", + "start_time": "1696421866", + "uuid": "589966AE-074A-503B-B17B-54B05684A120", + "version": "5.9.1", + "watcher": "96729" + }`), + }, + }, + expected: []HostQueryResultRow{ + { + HostID: 1, + Hostname: "macOS host", + LastFetched: macOSOsqueryInfoLastFetched, + Columns: map[string]string{ + "build_distro": "10.14", + "build_platform": "darwin", + "config_hash": "eed0d8296e5f90b790a23814a9db7a127b13498d", + "config_valid": "1", + "extensions": "active", + "instance_id": "7f02ff0f-f8a7-4ba9-a1d2-66836b154f4a", + "pid": "96730", + "platform_mask": "21", + "start_time": "1696421866", + "uuid": "589966AE-074A-503B-B17B-54B05684A120", + "version": "5.9.1", + "watcher": "96729", + }, + }, + }, + shouldFail: false, + }, + { + name: "invalid JSON result", + rows: []*ScheduledQueryResultRow{ + { + HostID: 3, + Hostname: "bar", + LastFetched: time.Now(), + Data: json.RawMessage(`invalid JSON`), + }, + }, + shouldFail: true, + }, + { + name: "invalid item value type", + rows: []*ScheduledQueryResultRow{ + { + HostID: 3, + Hostname: "bar", + LastFetched: time.Now(), + Data: json.RawMessage(`{"foobar": 1}`), + }, + }, + shouldFail: true, + }, + } { + results, err := MapQueryReportResultsToRows(tc.rows) + if !tc.shouldFail { + require.NoError(t, err) + require.Equal(t, tc.expected, results) + } else { + require.Error(t, err) + } + } +} diff --git a/server/fleet/service.go b/server/fleet/service.go index 9554c78154..8967dc0408 100644 --- a/server/fleet/service.go +++ b/server/fleet/service.go @@ -268,6 +268,8 @@ type Service interface { // and only non-scheduled queries will be returned if `*scheduled == false`. ListQueries(ctx context.Context, opt ListOptions, teamID *uint, scheduled *bool) ([]*Query, error) GetQuery(ctx context.Context, id uint) (*Query, error) + // GetQueryReportResults returns all the stored results of a query. + GetQueryReportResults(ctx context.Context, id uint) ([]HostQueryResultRow, error) NewQuery(ctx context.Context, p QueryPayload) (*Query, error) ModifyQuery(ctx context.Context, id uint, p QueryPayload) (*Query, error) DeleteQuery(ctx context.Context, teamID *uint, name string) error @@ -451,6 +453,7 @@ type Service interface { ScheduleQuery(ctx context.Context, sq *ScheduledQuery) (query *ScheduledQuery, err error) DeleteScheduledQuery(ctx context.Context, id uint) (err error) ModifyScheduledQuery(ctx context.Context, id uint, p ScheduledQueryPayload) (query *ScheduledQuery, err error) + SaveResultLogsToQueryReports(ctx context.Context, results []json.RawMessage) // ///////////////////////////////////////////////////////////////////////////// // StatusService diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 7d66437592..89a41b413e 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -294,6 +294,14 @@ type CleanupExpiredHostsFunc func(ctx context.Context) ([]uint, error) type ScheduledQueryIDsByNameFunc func(ctx context.Context, batchSize int, packAndSchedQueryNames ...[2]string) ([]uint, error) +type QueryResultRowsFunc func(ctx context.Context, queryID uint) ([]*fleet.ScheduledQueryResultRow, error) + +type ResultCountForQueryFunc func(ctx context.Context, queryID uint) (int, error) + +type ResultCountForQueryAndHostFunc func(ctx context.Context, queryID uint, hostID uint) (int, error) + +type OverwriteQueryResultRowsFunc func(ctx context.Context, rows []*fleet.ScheduledQueryResultRow) error + type NewTeamFunc func(ctx context.Context, team *fleet.Team) (*fleet.Team, error) type SaveTeamFunc func(ctx context.Context, team *fleet.Team) (*fleet.Team, error) @@ -1097,6 +1105,18 @@ type DataStore struct { ScheduledQueryIDsByNameFunc ScheduledQueryIDsByNameFunc ScheduledQueryIDsByNameFuncInvoked bool + QueryResultRowsFunc QueryResultRowsFunc + QueryResultRowsFuncInvoked bool + + ResultCountForQueryFunc ResultCountForQueryFunc + ResultCountForQueryFuncInvoked bool + + ResultCountForQueryAndHostFunc ResultCountForQueryAndHostFunc + ResultCountForQueryAndHostFuncInvoked bool + + OverwriteQueryResultRowsFunc OverwriteQueryResultRowsFunc + OverwriteQueryResultRowsFuncInvoked bool + NewTeamFunc NewTeamFunc NewTeamFuncInvoked bool @@ -2648,6 +2668,34 @@ func (s *DataStore) ScheduledQueryIDsByName(ctx context.Context, batchSize int, return s.ScheduledQueryIDsByNameFunc(ctx, batchSize, packAndSchedQueryNames...) } +func (s *DataStore) QueryResultRows(ctx context.Context, queryID uint) ([]*fleet.ScheduledQueryResultRow, error) { + s.mu.Lock() + s.QueryResultRowsFuncInvoked = true + s.mu.Unlock() + return s.QueryResultRowsFunc(ctx, queryID) +} + +func (s *DataStore) ResultCountForQuery(ctx context.Context, queryID uint) (int, error) { + s.mu.Lock() + s.ResultCountForQueryFuncInvoked = true + s.mu.Unlock() + return s.ResultCountForQueryFunc(ctx, queryID) +} + +func (s *DataStore) ResultCountForQueryAndHost(ctx context.Context, queryID uint, hostID uint) (int, error) { + s.mu.Lock() + s.ResultCountForQueryAndHostFuncInvoked = true + s.mu.Unlock() + return s.ResultCountForQueryAndHostFunc(ctx, queryID, hostID) +} + +func (s *DataStore) OverwriteQueryResultRows(ctx context.Context, rows []*fleet.ScheduledQueryResultRow) error { + s.mu.Lock() + s.OverwriteQueryResultRowsFuncInvoked = true + s.mu.Unlock() + return s.OverwriteQueryResultRowsFunc(ctx, rows) +} + func (s *DataStore) NewTeam(ctx context.Context, team *fleet.Team) (*fleet.Team, error) { s.mu.Lock() s.NewTeamFuncInvoked = true diff --git a/server/service/handler.go b/server/service/handler.go index aa76a30afa..91c7219e1b 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -338,6 +338,7 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC ue.GET("/api/_version_/fleet/queries/{id:[0-9]+}", getQueryEndpoint, getQueryRequest{}) ue.GET("/api/_version_/fleet/queries", listQueriesEndpoint, listQueriesRequest{}) + ue.GET("/api/_version_/fleet/queries/{id:[0-9]+}/report", getQueryReportEndpoint, getQueryReportRequest{}) ue.POST("/api/_version_/fleet/queries", createQueryEndpoint, createQueryRequest{}) ue.PATCH("/api/_version_/fleet/queries/{id:[0-9]+}", modifyQueryEndpoint, modifyQueryRequest{}) ue.DELETE("/api/_version_/fleet/queries/{name}", deleteQueryEndpoint, deleteQueryRequest{}) diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index 89650857d8..ca2b669ac1 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -7904,3 +7904,283 @@ func (s *integrationTestSuite) TestHostsReportWithPolicyResults() { }) } } + +func (s *integrationTestSuite) TestQueryReports() { + t := s.T() + ctx := context.Background() + + team1, err := s.ds.NewTeam(ctx, &fleet.Team{ + ID: 42, + Name: "team1", + Description: "desc team1", + }) + require.NoError(t, err) + + host1Global, err := s.ds.NewHost(ctx, &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: ptr.String("1"), + UUID: "1", + Hostname: "foo.local1", + OsqueryHostID: ptr.String("1"), + PrimaryIP: "192.168.1.1", + PrimaryMac: "30-65-EC-6F-C4-58", + Platform: "ubuntu", + }) + require.NoError(t, err) + + host2Team1, err := s.ds.NewHost(ctx, &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: ptr.String("2"), + UUID: "2", + Hostname: "foo.local2", + OsqueryHostID: ptr.String("2"), + PrimaryIP: "192.168.1.2", + PrimaryMac: "30-65-EC-6F-C4-59", + Platform: "darwin", + }) + require.NoError(t, err) + + err = s.ds.AddHostsToTeam(ctx, &team1.ID, []uint{host2Team1.ID}) + require.NoError(t, err) + + osqueryInfoQuery, err := s.ds.NewQuery(ctx, &fleet.Query{ + Name: "Osquery info", + Description: "osquery_info table", + Query: "select * from osquery_info;", + Saved: true, + Interval: 30, + AutomationsEnabled: true, + DiscardData: false, + TeamID: nil, + Logging: fleet.LoggingSnapshot, + }) + require.NoError(t, err) + + usbDevicesQuery, err := s.ds.NewQuery(ctx, &fleet.Query{ + Name: "USB devices", + Description: "usb_devices table", + Query: "select * from usb_devices;", + Saved: true, + Interval: 60, + AutomationsEnabled: true, + DiscardData: false, + TeamID: ptr.Uint(team1.ID), + Logging: fleet.LoggingSnapshot, + }) + require.NoError(t, err) + + slreq := submitLogsRequest{ + NodeKey: *host2Team1.NodeKey, + LogType: "result", + Data: json.RawMessage(`[{ + "snapshot": [ + { + "class": "239", + "model": "HD Pro Webcam C920", + "model_id": "0892", + "protocol": "", + "removable": "1", + "serial": "zoobar", + "subclass": "2", + "usb_address": "3", + "usb_port": "1", + "vendor": "", + "vendor_id": "046d", + "version": "0.19" + }, + { + "class": "0", + "model": "Apple Internal Keyboard / Trackpad", + "model_id": "027e", + "protocol": "", + "removable": "0", + "serial": "foobar", + "subclass": "0", + "usb_address": "8", + "usb_port": "5", + "vendor": "Apple Inc.", + "vendor_id": "05ac", + "version": "9.33" + } + ], + "action": "snapshot", + "name": "pack/team-` + usbDevicesQuery.TeamIDStr() + `/` + usbDevicesQuery.Name + `", + "hostIdentifier": "` + *host2Team1.OsqueryHostID + `", + "calendarTime": "Fri Oct 6 17:32:08 2023 UTC", + "unixTime": 1696613528, + "epoch": 0, + "counter": 0, + "numerics": false, + "decorations": { + "host_uuid": "` + host2Team1.UUID + `", + "hostname": "` + host2Team1.Hostname + `" + } +}, +{ + "snapshot": [ + { + "build_distro": "10.14", + "build_platform": "darwin", + "config_hash": "eed0d8296e5f90b790a23814a9db7a127b13498d", + "config_valid": "1", + "extensions": "active", + "instance_id": "7f02ff0f-f8a7-4ba9-a1d2-66836b154f4a", + "pid": "95637", + "platform_mask": "21", + "start_time": "1696611201", + "uuid": "` + host2Team1.UUID + `", + "version": "5.9.1", + "watcher": "95636" + } + ], + "action": "snapshot", + "name": "pack/Global/` + osqueryInfoQuery.Name + `", + "hostIdentifier": "` + *host2Team1.OsqueryHostID + `", + "calendarTime": "Fri Oct 6 18:08:18 2023 UTC", + "unixTime": 1696615698, + "epoch": 0, + "counter": 0, + "numerics": false, + "decorations": { + "host_uuid": "` + host2Team1.UUID + `", + "hostname": "` + host2Team1.Hostname + `" + } +} +]`), + } + slres := submitLogsResponse{} + s.DoJSON("POST", "/api/osquery/log", slreq, http.StatusOK, &slres) + require.NoError(t, slres.Err) + + slreq = submitLogsRequest{ + NodeKey: *host1Global.NodeKey, + LogType: "result", + Data: json.RawMessage(`[{ + "snapshot": [ + { + "build_distro": "centos7", + "build_platform": "linux", + "config_hash": "eed0d8296e5f90b790a23814a9db7a127b13498d", + "config_valid": "1", + "extensions": "active", + "instance_id": "e5799132-85ab-4cfa-89f3-03e0dd3c509a", + "pid": "3574", + "platform_mask": "9", + "start_time": "1696502961", + "uuid": "` + host1Global.UUID + `", + "version": "5.9.2", + "watcher": "3570" + } + ], + "action": "snapshot", + "name": "pack/Global/` + osqueryInfoQuery.Name + `", + "hostIdentifier": "` + *host1Global.OsqueryHostID + `", + "calendarTime": "Fri Oct 6 18:13:04 2023 UTC", + "unixTime": 1696615984, + "epoch": 0, + "counter": 0, + "numerics": false, + "decorations": { + "host_uuid": "187c4d56-8e45-1a9d-8513-ac17efd2f0fd", + "hostname": "` + host1Global.Hostname + `" + } +}]`), + } + slres = submitLogsResponse{} + s.DoJSON("POST", "/api/osquery/log", slreq, http.StatusOK, &slres) + require.NoError(t, slres.Err) + + var gqrr getQueryReportResponse + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/queries/%d/report", usbDevicesQuery.ID), getQueryReportRequest{}, http.StatusOK, &gqrr) + require.NoError(t, gqrr.Err) + require.Equal(t, usbDevicesQuery.ID, gqrr.QueryID) + require.Len(t, gqrr.Results, 2) + sort.Slice(gqrr.Results, func(i, j int) bool { + // Let's just pick a known column of the query to sort. + return gqrr.Results[i].Columns["usb_port"] < gqrr.Results[j].Columns["usb_port"] + }) + require.Equal(t, host2Team1.ID, gqrr.Results[0].HostID) + require.Equal(t, host2Team1.Hostname, gqrr.Results[0].Hostname) + require.NotZero(t, gqrr.Results[0].LastFetched) + require.Equal(t, map[string]string{ + "class": "239", + "model": "HD Pro Webcam C920", + "model_id": "0892", + "protocol": "", + "removable": "1", + "serial": "zoobar", + "subclass": "2", + "usb_address": "3", + "usb_port": "1", + "vendor": "", + "vendor_id": "046d", + "version": "0.19", + }, gqrr.Results[0].Columns) + require.Equal(t, host2Team1.ID, gqrr.Results[1].HostID) + require.Equal(t, host2Team1.Hostname, gqrr.Results[1].Hostname) + require.NotZero(t, gqrr.Results[1].LastFetched) + require.Equal(t, map[string]string{ + "class": "0", + "model": "Apple Internal Keyboard / Trackpad", + "model_id": "027e", + "protocol": "", + "removable": "0", + "serial": "foobar", + "subclass": "0", + "usb_address": "8", + "usb_port": "5", + "vendor": "Apple Inc.", + "vendor_id": "05ac", + "version": "9.33", + }, gqrr.Results[1].Columns) + + gqrr = getQueryReportResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/queries/%d/report", osqueryInfoQuery.ID), getQueryReportRequest{}, http.StatusOK, &gqrr) + require.NoError(t, gqrr.Err) + require.Equal(t, osqueryInfoQuery.ID, gqrr.QueryID) + require.Len(t, gqrr.Results, 2) + sort.Slice(gqrr.Results, func(i, j int) bool { + // Let's just pick a known column of the query to sort. + return gqrr.Results[i].Columns["version"] > gqrr.Results[j].Columns["version"] + }) + require.Equal(t, host1Global.ID, gqrr.Results[0].HostID) + require.Equal(t, host1Global.Hostname, gqrr.Results[0].Hostname) + require.NotZero(t, gqrr.Results[0].LastFetched) + require.Equal(t, map[string]string{ + "build_distro": "centos7", + "build_platform": "linux", + "config_hash": "eed0d8296e5f90b790a23814a9db7a127b13498d", + "config_valid": "1", + "extensions": "active", + "instance_id": "e5799132-85ab-4cfa-89f3-03e0dd3c509a", + "pid": "3574", + "platform_mask": "9", + "start_time": "1696502961", + "uuid": host1Global.UUID, + "version": "5.9.2", + "watcher": "3570", + }, gqrr.Results[0].Columns) + require.Equal(t, host2Team1.ID, gqrr.Results[1].HostID) + require.Equal(t, host2Team1.Hostname, gqrr.Results[1].Hostname) + require.NotZero(t, gqrr.Results[1].LastFetched) + require.Equal(t, map[string]string{ + "build_distro": "10.14", + "build_platform": "darwin", + "config_hash": "eed0d8296e5f90b790a23814a9db7a127b13498d", + "config_valid": "1", + "extensions": "active", + "instance_id": "7f02ff0f-f8a7-4ba9-a1d2-66836b154f4a", + "pid": "95637", + "platform_mask": "21", + "start_time": "1696611201", + "uuid": host2Team1.UUID, + "version": "5.9.1", + "watcher": "95636", + }, gqrr.Results[1].Columns) +} diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 83f4976de8..8af523353e 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -1898,21 +1898,26 @@ func (s *integrationEnterpriseTestSuite) TestListDevicePolicies() { // try with invalid token res := s.DoRawNoAuth("GET", "/api/latest/fleet/device/invalid_token/policies", nil, http.StatusUnauthorized) - res.Body.Close() + err = res.Body.Close() + require.NoError(t, err) // GET `/api/_version_/fleet/device/{token}/policies` listDevicePoliciesResp := listDevicePoliciesResponse{} res = s.DoRawNoAuth("GET", "/api/latest/fleet/device/"+token+"/policies", nil, http.StatusOK) - json.NewDecoder(res.Body).Decode(&listDevicePoliciesResp) //nolint:errcheck - res.Body.Close() //nolint:errcheck + err = json.NewDecoder(res.Body).Decode(&listDevicePoliciesResp) + require.NoError(t, err) + err = res.Body.Close() + require.NoError(t, err) require.Len(t, listDevicePoliciesResp.Policies, 2) require.NoError(t, listDevicePoliciesResp.Err) // GET `/api/_version_/fleet/device/{token}` getDeviceHostResp := getDeviceHostResponse{} res = s.DoRawNoAuth("GET", "/api/latest/fleet/device/"+token, nil, http.StatusOK) - json.NewDecoder(res.Body).Decode(&getDeviceHostResp) //nolint:errcheck - res.Body.Close() //nolint:errcheck + err = json.NewDecoder(res.Body).Decode(&getDeviceHostResp) + require.NoError(t, err) + err = res.Body.Close() + require.NoError(t, err) require.NoError(t, getDeviceHostResp.Err) require.Equal(t, host.ID, getDeviceHostResp.Host.ID) require.False(t, getDeviceHostResp.Host.RefetchRequested) @@ -1922,8 +1927,10 @@ func (s *integrationEnterpriseTestSuite) TestListDevicePolicies() { // GET `/api/_version_/fleet/device/{token}/desktop` getDesktopResp := fleetDesktopResponse{} res = s.DoRawNoAuth("GET", "/api/latest/fleet/device/"+token+"/desktop", nil, http.StatusOK) - require.NoError(t, json.NewDecoder(res.Body).Decode(&getDesktopResp)) - require.NoError(t, res.Body.Close()) + err = json.NewDecoder(res.Body).Decode(&getDesktopResp) + require.NoError(t, err) + err = res.Body.Close() + require.NoError(t, err) require.NoError(t, getDesktopResp.Err) require.Equal(t, *getDesktopResp.FailingPolicies, uint(1)) require.False(t, getDesktopResp.Notifications.NeedsMDMMigration) diff --git a/server/service/osquery.go b/server/service/osquery.go index 47852be0d0..e9ea4b604b 100644 --- a/server/service/osquery.go +++ b/server/service/osquery.go @@ -20,8 +20,8 @@ import ( "github.com/fleetdm/fleet/v4/server/ptr" "github.com/fleetdm/fleet/v4/server/pubsub" "github.com/fleetdm/fleet/v4/server/service/osquery_utils" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/spf13/cast" ) @@ -1358,6 +1358,9 @@ func submitLogsEndpoint(ctx context.Context, request interface{}, svc fleet.Serv switch req.LogType { case "status": var statuses []json.RawMessage + // NOTE(lucas): This unmarshal error is not being sent back to osquery (`if err :=` vs. `if err =`) + // Maybe there's a reason for it, we need to test such a change before fixing what appears + // to be a bug because the `err` is lost. if err := json.Unmarshal(req.Data, &statuses); err != nil { err = newOsqueryError("unmarshalling status logs: " + err.Error()) break @@ -1370,12 +1373,18 @@ func submitLogsEndpoint(ctx context.Context, request interface{}, svc fleet.Serv case "result": var results []json.RawMessage + // NOTE(lucas): This unmarshal error is not being sent back to osquery (`if err :=` vs. `if err =`) + // Maybe there's a reason for it, we need to test such a change before fixing what appears + // to be a bug because the `err` is lost. if err := json.Unmarshal(req.Data, &results); err != nil { err = newOsqueryError("unmarshalling result logs: " + err.Error()) break } - err = svc.SubmitResultLogs(ctx, results) - if err != nil { + + // Not returning errors as it will trigger osqueryd to retry the request + svc.SaveResultLogsToQueryReports(ctx, results) + + if err = svc.SubmitResultLogs(ctx, results); err != nil { break } @@ -1405,3 +1414,167 @@ func (svc *Service) SubmitResultLogs(ctx context.Context, logs []json.RawMessage } return nil } + +//////////////////////////////////////////////////////////////////////////////// +// Query Reports +//////////////////////////////////////////////////////////////////////////////// + +func (svc *Service) SaveResultLogsToQueryReports(ctx context.Context, results []json.RawMessage) { + // skipauth: Authorization is currently for user endpoints only. + svc.authz.SkipAuthorization(ctx) + + // Do not insert results if query reports are disabled globally + appConfig, err := svc.ds.AppConfig(ctx) + if err != nil { + level.Error(svc.logger).Log("err", "getting app config", "err", err) + return + } + if appConfig.ServerSettings.QueryReportsDisabled { + return + } + + var queryResults []fleet.ScheduledQueryResult + for _, raw := range results { + var result fleet.ScheduledQueryResult + if err := json.Unmarshal(raw, &result); err != nil { + level.Error(svc.logger).Log("err", "unmarshalling result", "err", err) + continue + } + queryResults = append(queryResults, result) + } + + // Filter results to only the most recent for each query + filtered := getMostRecentResults(queryResults) + + for _, result := range filtered { + if err := svc.processResults(ctx, result); err != nil { + level.Error(svc.logger).Log("err", "processing result", "err", err) + continue + } + } + + return +} + +func (svc *Service) processResults(ctx context.Context, result fleet.ScheduledQueryResult) error { + // Discard result if there is no snapshot + if len(result.Snapshot) == 0 { + return nil + } + + teamID, queryName, err := getQueryNameAndTeamIDFromResult(result.QueryName) + if err != nil { + return ctxerr.Wrap(ctx, err, "querying name and team ID from result") + } + + query, err := svc.ds.QueryByName(ctx, teamID, queryName) + if err != nil { + return nil // not logging here due to a known issue when renaming queries + } + + // Discard Result if query is marked as discard data or if logging is not snapshot + if query.DiscardData || query.Logging != fleet.LoggingSnapshot { + return nil + } + + host, ok := hostctx.FromContext(ctx) + if !ok { + return ctxerr.Wrap(ctx, err, "getting host from context") + } + + return svc.overwriteResultRows(ctx, result, query.ID, host.ID) +} + +// The "snapshot" array in a ScheduledQueryResult can contain multiple rows. Each +// row is saved as a separate ScheduledQueryResultRow. ie. a result could contain +// many USB Devices or a result could contain all User Accounts on a host. +func (svc *Service) overwriteResultRows(ctx context.Context, result fleet.ScheduledQueryResult, queryID, hostID uint) error { + fetchTime := time.Now() + + rows := make([]*fleet.ScheduledQueryResultRow, 0, len(result.Snapshot)) + + for _, snapshotItem := range result.Snapshot { + + row := &fleet.ScheduledQueryResultRow{ + QueryID: queryID, + HostID: hostID, + Data: snapshotItem, + LastFetched: fetchTime, + } + + rows = append(rows, row) + + } + + if err := svc.ds.OverwriteQueryResultRows(ctx, rows); err != nil { + return ctxerr.Wrap(ctx, err, "overwriting query result rows") + } + + return nil +} + +// getMostRecentResults returns only the most recent result per query. +// Osquery can send multiple results for the same query (ie. if an agent loses +// network connectivity it will cache multiple results). Query Reports only +// save the most recent result for a given query. +func getMostRecentResults(results []fleet.ScheduledQueryResult) []fleet.ScheduledQueryResult { + // Use a map to track the most recent entry for each unique QueryName + latestResults := make(map[string]fleet.ScheduledQueryResult) + + for _, result := range results { + if existing, ok := latestResults[result.QueryName]; ok { + // Compare the UnixTime time and update the map if the current result is more recent + if result.UnixTime > existing.UnixTime { + latestResults[result.QueryName] = result + } + } else { + latestResults[result.QueryName] = result + } + } + + // Convert the map back to a slice + var filteredResults []fleet.ScheduledQueryResult + for _, v := range latestResults { + filteredResults = append(filteredResults, v) + } + + return filteredResults +} + +// Query names recieved from osqueryd are prefixed by teamID so we need +// to pull them out to match the query name and team ID in the database +func getQueryNameAndTeamIDFromResult(path string) (*uint, string, error) { + // For pattern: pack/Global/Name + if strings.HasPrefix(path, "pack/Global/") { + return nil, strings.TrimPrefix(path, "pack/Global/"), nil + } + + // For pattern: pack/team-/Name + if strings.HasPrefix(path, "pack/team-") { + parts := strings.SplitN(path, "/", 3) + if len(parts) != 3 { + return nil, "", fmt.Errorf("unknown format: %s", path) + } + + teamNumberStr := strings.TrimPrefix(parts[1], "team-") + teamNumberUint, err := strconv.ParseUint(teamNumberStr, 10, 32) + if err != nil { + return nil, "", fmt.Errorf("parsing team number: %w", err) + } + + teamNumber := uint(teamNumberUint) + return &teamNumber, parts[2], nil + } + + // For pattern: pack/PackName/Query (legacy pack) + if strings.HasPrefix(path, "pack/") { + parts := strings.SplitN(path, "/", 3) + if len(parts) != 3 { + return nil, "", fmt.Errorf("unknown format: %s", path) + } + return nil, parts[2], nil + } + + // If none of the above patterns match, return error + return nil, "", fmt.Errorf("unknown format: %s", path) +} diff --git a/server/service/osquery_test.go b/server/service/osquery_test.go index a0da69b815..223df9fd12 100644 --- a/server/service/osquery_test.go +++ b/server/service/osquery_test.go @@ -563,6 +563,135 @@ func TestSubmitResultLogs(t *testing.T) { assert.Equal(t, results, testLogger.logs) } +func TestSaveResultLogsToQueryReports(t *testing.T) { + ds := new(mock.Store) + svc, ctx := newTestService(t, ds, nil, nil) + + logRawMessages := []json.RawMessage{ + json.RawMessage(`{"snapshot":[{"hour":"20","minutes":"8"}],"action":"snapshot","name":"pack/Global/Uptime","hostIdentifier":"1379f59d98f4","calendarTime":"Tue Jan 10 20:08:51 2017 UTC","unixTime":1484078931,"decorations":{"host_uuid":"EB714C9D-C1F8-A436-B6DA-3F853C5502EA"}}`), + } + + host := fleet.Host{} + ctx = hostctx.NewContext(ctx, &host) + + // Results not saved if query reports disabled globally + ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { + return &fleet.AppConfig{ServerSettings: fleet.ServerSettings{QueryReportsDisabled: true}}, nil + } + svc.SaveResultLogsToQueryReports(ctx, logRawMessages) + + // Result not saved if result is not a snapshot + logRawMessages = []json.RawMessage{ + json.RawMessage(`{"name":"pack/Global/Uptime","hostIdentifier":"2e23c347-da72-4e72-b6a8-a6b8a9a46ab7","calendarTime":"Fri Oct 6 14:19:15 2023 UTC","unixTime":1696601955,"epoch":0,"counter":10,"numerics":false,"decorations":{"host_uuid":"550eb898-c522-410b-8855-d74d94fdfcd2","hostname":"0025ad6e71fb"},"columns":{"days":"0","hours":"4","minutes":"52","seconds":"25","total_seconds":"17545"},"action":"removed"}`), + } + svc.SaveResultLogsToQueryReports(ctx, logRawMessages) + + // Results not saved if Logging is not snapshot in the query config + logRawMessages = []json.RawMessage{ + json.RawMessage(`{"snapshot":[{"hour":"20","minutes":"8"}],"action":"snapshot","name":"pack/Global/Uptime","hostIdentifier":"1379f59d98f4","calendarTime":"Tue Jan 10 20:08:51 2017 UTC","unixTime":1484078931,"decorations":{"host_uuid":"EB714C9D-C1F8-A436-B6DA-3F853C5502EA"}}`), + } + + ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { + return &fleet.AppConfig{ServerSettings: fleet.ServerSettings{QueryReportsDisabled: false}}, nil + } + + // Results not saved if DiscardData is true in Query + ds.QueryByNameFunc = func(ctx context.Context, teamID *uint, name string, opts ...fleet.OptionalArg) (*fleet.Query, error) { + return &fleet.Query{ID: 1, DiscardData: true, Logging: fleet.LoggingSnapshot}, nil + } + + svc.SaveResultLogsToQueryReports(ctx, logRawMessages) + + // Happy Path: Results saved + ds.QueryByNameFunc = func(ctx context.Context, teamID *uint, name string, opts ...fleet.OptionalArg) (*fleet.Query, error) { + return &fleet.Query{ID: 1, DiscardData: false, Logging: fleet.LoggingSnapshot}, nil + } + ds.OverwriteQueryResultRowsFunc = func(ctx context.Context, rows []*fleet.ScheduledQueryResultRow) error { + return nil + } + svc.SaveResultLogsToQueryReports(ctx, logRawMessages) + require.True(t, ds.OverwriteQueryResultRowsFuncInvoked) +} + +func TestGetQueryNameAndTeamIDFromResult(t *testing.T) { + tests := []struct { + input string + expectedID *uint + expectedName string + hasErr bool + }{ + {"pack/Global/Query Name", nil, "Query Name", false}, + {"pack/team-1/Query Name", ptr.Uint(1), "Query Name", false}, + {"pack/team-12345/Another Query", ptr.Uint(12345), "Another Query", false}, + {"pack/PackName/Query", nil, "Query", false}, // Legacy Pack support + {"pack/team-foo/Query", nil, "", true}, + {"pack/Global/QueryWith/Slash", nil, "QueryWith/Slash", false}, + {"pack/team-1/QueryWith/Slash", ptr.Uint(1), "QueryWith/Slash", false}, + {"pack/PackName/QueryWith/Slash", nil, "QueryWith/Slash", false}, // Legacy Pack support + {"InvalidString", nil, "", true}, + {"Invalid/Query", nil, "", true}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + id, str, err := getQueryNameAndTeamIDFromResult(tt.input) + assert.Equal(t, tt.expectedID, id) + assert.Equal(t, tt.expectedName, str) + if tt.hasErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestGetMostRecentResults(t *testing.T) { + tests := []struct { + name string + input []fleet.ScheduledQueryResult + expected []fleet.ScheduledQueryResult + }{ + { + name: "basic test", + input: []fleet.ScheduledQueryResult{ + {QueryName: "test1", UnixTime: 1}, + {QueryName: "test1", UnixTime: 2}, + {QueryName: "test1", UnixTime: 3}, + {QueryName: "test2", UnixTime: 1}, + {QueryName: "test2", UnixTime: 2}, + {QueryName: "test2", UnixTime: 3}, + }, + expected: []fleet.ScheduledQueryResult{ + {QueryName: "test1", UnixTime: 3}, + {QueryName: "test2", UnixTime: 3}, + }, + }, + { + name: "out of order test", + input: []fleet.ScheduledQueryResult{ + {QueryName: "test1", UnixTime: 2}, + {QueryName: "test1", UnixTime: 3}, + {QueryName: "test1", UnixTime: 1}, + {QueryName: "test2", UnixTime: 3}, + {QueryName: "test2", UnixTime: 2}, + {QueryName: "test2", UnixTime: 1}, + }, + expected: []fleet.ScheduledQueryResult{ + {QueryName: "test1", UnixTime: 3}, + {QueryName: "test2", UnixTime: 3}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + results := getMostRecentResults(tt.input) + assert.Equal(t, tt.expected, results) + }) + } +} + func verifyDiscovery(t *testing.T, queries, discovery map[string]string) { assert.Equal(t, len(queries), len(discovery)) // discoveryUsed holds the queries where we know use the distributed discovery feature. diff --git a/server/service/queries.go b/server/service/queries.go index 7a59033790..77abcde504 100644 --- a/server/service/queries.go +++ b/server/service/queries.go @@ -122,6 +122,56 @@ func onlyShowObserverCanRunQueries(user *fleet.User, teamID *uint) bool { })[*teamID] } +//////////////////////////////////////////////////////////////////////////////// +// Get query report +//////////////////////////////////////////////////////////////////////////////// + +type getQueryReportRequest struct { + ID uint `url:"id"` +} + +type getQueryReportResponse struct { + QueryID uint `json:"query_id"` + Results []fleet.HostQueryResultRow `json:"results"` + Err error `json:"error,omitempty"` +} + +func (r getQueryReportResponse) error() error { return r.Err } + +func getQueryReportEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) { + req := request.(*getQueryReportRequest) + queryReportResults, err := svc.GetQueryReportResults(ctx, req.ID) + if err != nil { + return listQueriesResponse{Err: err}, nil + } + return getQueryReportResponse{ + QueryID: req.ID, + Results: queryReportResults, + }, nil +} + +func (svc *Service) GetQueryReportResults(ctx context.Context, id uint) ([]fleet.HostQueryResultRow, error) { + // Load query first to get its teamID. + query, err := svc.ds.Query(ctx, id) + if err != nil { + setAuthCheckedOnPreAuthErr(ctx) + return nil, ctxerr.Wrap(ctx, err, "get query from datastore") + } + if err := svc.authz.Authorize(ctx, query, fleet.ActionRead); err != nil { + return nil, err + } + + queryReportResultRows, err := svc.ds.QueryResultRows(ctx, id) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "get query report results") + } + queryReportResults, err := fleet.MapQueryReportResultsToRows(queryReportResultRows) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "map db rows to results") + } + return queryReportResults, nil +} + //////////////////////////////////////////////////////////////////////////////// // Create Query //////////////////////////////////////////////////////////////////////////////// @@ -193,6 +243,9 @@ func (svc *Service) NewQuery(ctx context.Context, p fleet.QueryPayload) (*fleet. if p.ObserverCanRun != nil { query.ObserverCanRun = *p.ObserverCanRun } + if p.DiscardData != nil { + query.DiscardData = *p.DiscardData + } logging.WithExtras(ctx, "name", query.Name, "sql", query.Query) @@ -594,6 +647,7 @@ func (svc *Service) queryFromSpec(ctx context.Context, spec *fleet.QuerySpec) (* MinOsqueryVersion: spec.MinOsqueryVersion, AutomationsEnabled: spec.AutomationsEnabled, Logging: spec.Logging, + DiscardData: spec.DiscardData, }, nil } @@ -664,6 +718,7 @@ func (svc *Service) specFromQuery(ctx context.Context, query *fleet.Query) (*fle MinOsqueryVersion: query.MinOsqueryVersion, AutomationsEnabled: query.AutomationsEnabled, Logging: query.Logging, + DiscardData: query.DiscardData, }, nil }