From 330088aeba7d2f92492c7f2bc45381b82a01036e Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Thu, 18 Jan 2024 12:41:06 -0300 Subject: [PATCH] Add support for custom `pack_delimiter` to query reports (#16162) #15490 - [X] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - [X] Added/updated tests - [X] Manual QA for all new/changed functionality --- ...uery-reports-support-custom-pack_delimiter | 1 + server/service/osquery.go | 67 ++++++++++++++----- server/service/osquery_test.go | 29 ++++++++ 3 files changed, 80 insertions(+), 17 deletions(-) create mode 100644 changes/15490-query-reports-support-custom-pack_delimiter diff --git a/changes/15490-query-reports-support-custom-pack_delimiter b/changes/15490-query-reports-support-custom-pack_delimiter new file mode 100644 index 0000000000..ddf30157f1 --- /dev/null +++ b/changes/15490-query-reports-support-custom-pack_delimiter @@ -0,0 +1 @@ +* Query reports feature now supports a custom `pack_delimiter` in the agent settings. diff --git a/server/service/osquery.go b/server/service/osquery.go index 5fb45bab0d..3840149be0 100644 --- a/server/service/osquery.go +++ b/server/service/osquery.go @@ -1703,35 +1703,68 @@ func getMostRecentResults(results []*fleet.ScheduledQueryResult) []*fleet.Schedu return filteredResults } -// Query names recieved from osqueryd are prefixed by teamID so we need -// to pull them out to match the query name and team ID in the database +// findPackDelimiterString attempts to find the `pack_delimiter` string in the scheduled +// query name reported by osquery (note that `pack_delimiter` can contain multiple characters). +// +// The expected format for s is "pack{Global|team-}" +// +// Returns "" if it failed to parse the pack_delimiter. +func findPackDelimiterString(scheduledQueryName string) string { + // Go's regexp doesn't support backreferences so we have to perform some manual work. + scheduledQueryName = scheduledQueryName[4:] // always starts with "pack" + for l := 1; l < len(scheduledQueryName); l++ { + sep := scheduledQueryName[:l] + rest := scheduledQueryName[l:] + pattern := fmt.Sprintf(`^(?:(Global)|(team-\d+))%s.+`, regexp.QuoteMeta(sep)) + matched, _ := regexp.MatchString(pattern, rest) + if matched { + return sep + } + } + return "" +} + +// getQueryNameAndTeamIDFromResult attempts to parse the scheduled query name reported by osquery. +// +// The expected format of query names managed by Fleet is: +// "pack{Global|team-}" func getQueryNameAndTeamIDFromResult(path string) (*uint, string, error) { + if !strings.HasPrefix(path, "pack") || len(path) <= 4 { + return nil, "", fmt.Errorf("unknown format: %q", path) + } + + sep := findPackDelimiterString(path) + if sep == "" { + // If a pack_delimiter could not be parsed we return an error. + // + // 2017/legacy packs with the format "pack// are + // considered unknown format (they are not considered global or team + // scheduled queries). + return nil, "", fmt.Errorf("unknown format: %q", path) + } + // For pattern: pack/Global/Name - if strings.HasPrefix(path, "pack/Global/") { - return nil, strings.TrimPrefix(path, "pack/Global/"), nil + globalPattern := "pack" + sep + "Global" + sep + if strings.HasPrefix(path, globalPattern) { + return nil, strings.TrimPrefix(path, globalPattern), nil } // For pattern: pack/team-/Name - if strings.HasPrefix(path, "pack/team-") { - parts := strings.SplitN(path, "/", 3) - if len(parts) != 3 { - return nil, "", fmt.Errorf("unknown format: %q", path) + teamPattern := "pack" + sep + "team-" + if strings.HasPrefix(path, teamPattern) { + teamIDAndRest := strings.TrimPrefix(path, teamPattern) + teamIDAndQueryNameParts := strings.SplitN(teamIDAndRest, sep, 2) + if len(teamIDAndQueryNameParts) != 2 { + return nil, "", fmt.Errorf("parsing team number part: %s", path) } - - teamNumberStr := strings.TrimPrefix(parts[1], "team-") - teamNumberUint, err := strconv.ParseUint(teamNumberStr, 10, 32) + teamNumberUint, err := strconv.ParseUint(teamIDAndQueryNameParts[0], 10, 32) if err != nil { return nil, "", fmt.Errorf("parsing team number: %w", err) } - teamNumber := uint(teamNumberUint) - return &teamNumber, parts[2], nil + return &teamNumber, teamIDAndQueryNameParts[1], nil } - // 2017/legacy packs with the format "pack// are - // considered unknown format (they are not considered global or team - // scheduled queries). - // If none of the above patterns match, return error return nil, "", fmt.Errorf("unknown format: %q", path) } diff --git a/server/service/osquery_test.go b/server/service/osquery_test.go index 90ddb4af20..faacefd796 100644 --- a/server/service/osquery_test.go +++ b/server/service/osquery_test.go @@ -564,6 +564,13 @@ func TestSubmitResultLogsToLogDestination(t *testing.T) { AutomationsEnabled: true, Logging: fleet.LoggingSnapshot, }, nil + case teamID == nil && name == "query_should_be_saved_and_submitted_with_custom_pack_delimiter": + return &fleet.Query{ + ID: 1234, + Name: name, + AutomationsEnabled: true, + Logging: fleet.LoggingSnapshot, + }, nil case teamID == nil && name == "query_should_be_saved_but_not_submitted": return &fleet.Query{ ID: 444, @@ -628,6 +635,7 @@ func TestSubmitResultLogsToLogDestination(t *testing.T) { `{"diffResults":{"removed":[{"address":"127.0.0.1","hostnames":"kl.groob.io"}],"added":""},"name":"pack\/team-1/hosts","hostIdentifier":"FA01680E-98CA-5557-8F59-7716ECFEE964","calendarTime":"Sun Nov 19 00:02:08 2017 UTC","unixTime":1511049728,"epoch":"0","counter":"10","decorations":{"host_uuid":"FA01680E-98CA-5557-8F59-7716ECFEE964","hostname":"kl.groob.io"}}`, `{"snapshot":[{"hour":"20","minutes":"8"}],"action":"snapshot","name":"pack/Global/query_should_be_saved_and_submitted","hostIdentifier":"1379f59d98f4","calendarTime":"Tue Jan 10 20:08:51 2017 UTC","unixTime":1484078931,"decorations":{"host_uuid":"EB714C9D-C1F8-A436-B6DA-3F853C5502EA"}}`, + `{"snapshot":[{"hour":"20","minutes":"8"}],"action":"snapshot","name":"pack_Global_query_should_be_saved_and_submitted_with_custom_pack_delimiter","hostIdentifier":"1379f59d98f4","calendarTime":"Tue Jan 10 20:08:52 2017 UTC","unixTime":1484078932,"decorations":{"host_uuid":"EB714C9D-C1F8-A436-B6DA-3F853C5502EA"}}`, // Fleet doesn't know of this query, so this result should be streamed as is (This is to support streaming results for osquery nodes that are configured outside of Fleet, e.g. `--config_plugin=filesystem`). `{"snapshot":[{"hour":"20","minutes":"8"}],"action":"snapshot","name":"pack/Global/doesntexist","hostIdentifier":"1379f59d98f4","calendarTime":"Tue Jan 10 20:08:51 2017 UTC","unixTime":1484078931,"decorations":{"host_uuid":"EB714C9D-C1F8-A436-B6DA-3F853C5502EA"}}`, @@ -844,19 +852,40 @@ func TestGetQueryNameAndTeamIDFromResult(t *testing.T) { {"pack/team-12345/Another Query", ptr.Uint(12345), "Another Query", false}, {"pack/team-foo/Query", nil, "", true}, {"pack/Global/QueryWith/Slash", nil, "QueryWith/Slash", false}, + {"packGlobalGlobalGlobalGlobal", nil, "Global", false}, // pack_delimiter=Global + {"packXGlobalGlobalXGlobalQueryWith/Slash", nil, "QueryWith/Slash", false}, // pack_delimiter=XGlobal + {"pack//Global//QueryWith/Slash", nil, "QueryWith/Slash", false}, // pack_delimiter=// {"pack/team-1/QueryWith/Slash", ptr.Uint(1), "QueryWith/Slash", false}, + {"pack_team-1_QueryWith/Slash", ptr.Uint(1), "QueryWith/Slash", false}, + {"packFOOBARteam-1FOOBARQueryWith/Slash", ptr.Uint(1), "QueryWith/Slash", false}, // pack_delimiter=FOOBAR + {"pack123😁123team-1123😁123QueryWith/Slash", ptr.Uint(1), "QueryWith/Slash", false}, // pack_delimiter=123😁123 + {"pack(foo)team-1(foo)fo(o)bar", ptr.Uint(1), "fo(o)bar", false}, // pack_delimiter=(foo) + {"packteam-1team-1team-1team-1", ptr.Uint(1), "team-1", false}, // pack_delimiter=team-1 {"InvalidString", nil, "", true}, {"Invalid/Query", nil, "", true}, + {"pac", nil, "", true}, + {"pack", nil, "", true}, + {"pack/", nil, "", true}, + {"pack/Global", nil, "", true}, + {"pack/Global/", nil, "", true}, + {"pack/team/foo", nil, "", true}, + {"pack/team-123", nil, "", true}, + {"pack/team-/foo", nil, "", true}, + {"pack/team-123/", nil, "", true}, // Legacy 2017 packs should fail the parsing as they are separate // from global or team queries. {"pack/PackName/Query", nil, "", true}, {"pack/PackName/QueryWith/Slash", nil, "", true}, + {"packFOOBARPackNameFOOBARQueryWith/Slash", nil, "", true}, // pack_delimiter=FOOBAR } for _, tt := range tests { + tt := tt t.Run(tt.input, func(t *testing.T) { + t.Parallel() + id, str, err := getQueryNameAndTeamIDFromResult(tt.input) assert.Equal(t, tt.expectedID, id) assert.Equal(t, tt.expectedName, str)