From ba88a37a3a08c967aa33ded01c03d55ca3f9599f Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Thu, 5 Feb 2026 15:55:03 -0300 Subject: [PATCH] Authenticate carve block endpoint before parsing the "data" field (#39353) - [X] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files) for more information. - [X] If paths of existing endpoints are modified without backwards compatibility, checked the frontend/CLI for any necessary changes ## Testing - [X] Added/updated automated tests - [X] QA'd all new/changed functionality manually --------- Co-authored-by: Magnus Jensen --- ...e-carve-block-endpoint-before-parsing-data | 1 + cmd/fleet/serve.go | 2 +- server/contexts/carvestore/carvestore.go | 23 + server/datastore/mysql/carves.go | 2 +- server/fleet/datastore.go | 4 + server/platform/http/errors.go | 2 +- server/service/carves.go | 163 +++++ server/service/carves_test.go | 602 ++++++++++++++++++ server/service/handler.go | 18 +- server/service/handler_test.go | 4 +- server/service/integration_core_test.go | 62 +- server/service/testing_utils.go | 3 +- 12 files changed, 870 insertions(+), 16 deletions(-) create mode 100644 changes/14087-authenticate-carve-block-endpoint-before-parsing-data create mode 100644 server/contexts/carvestore/carvestore.go diff --git a/changes/14087-authenticate-carve-block-endpoint-before-parsing-data b/changes/14087-authenticate-carve-block-endpoint-before-parsing-data new file mode 100644 index 0000000000..eb70e8f823 --- /dev/null +++ b/changes/14087-authenticate-carve-block-endpoint-before-parsing-data @@ -0,0 +1 @@ +* Improved authentication of `POST /api/v1/osquery/carve/block` requests by parsing and validating `session_id` and `request_id` before processing `data`. diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 92f3d2c8d3..76a7f3a1e5 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -1330,7 +1330,7 @@ the way that the Fleet server works. } extra = append(extra, service.WithHTTPSigVerifier(httpSigVerifier)) - apiHandler = service.MakeHandler(svc, config, httpLogger, limiterStore, redisPool, + apiHandler = service.MakeHandler(svc, config, httpLogger, limiterStore, redisPool, carveStore, []endpointer.HandlerRoutesFunc{android_service.GetRoutes(svc, androidSvc), activityRoutes}, extra...) setupRequired, err := svc.SetupRequired(baseCtx) diff --git a/server/contexts/carvestore/carvestore.go b/server/contexts/carvestore/carvestore.go new file mode 100644 index 0000000000..2c392d0b15 --- /dev/null +++ b/server/contexts/carvestore/carvestore.go @@ -0,0 +1,23 @@ +package carvestore + +import ( + "context" + + "github.com/fleetdm/fleet/v4/server/fleet" +) + +type key int + +const carveStoreKey key = 0 + +func NewContext(ctx context.Context, svc fleet.CarveBySessionIder) context.Context { + return context.WithValue(ctx, carveStoreKey, svc) +} + +func FromContext(ctx context.Context) fleet.CarveBySessionIder { + svc, ok := ctx.Value(carveStoreKey).(fleet.CarveBySessionIder) + if !ok { + return nil + } + return svc +} diff --git a/server/datastore/mysql/carves.go b/server/datastore/mysql/carves.go index 0b61006a77..4fea620a17 100644 --- a/server/datastore/mysql/carves.go +++ b/server/datastore/mysql/carves.go @@ -198,7 +198,7 @@ func (ds *Datastore) CarveBySessionId(ctx context.Context, sessionId string) (*f var metadata fleet.CarveMetadata if err := sqlx.GetContext(ctx, ds.reader(ctx), &metadata, stmt, sessionId); err != nil { if err == sql.ErrNoRows { - return nil, ctxerr.Wrap(ctx, notFound("CarveBySessionId").WithName(sessionId)) + return nil, ctxerr.Wrap(ctx, notFound("CarveBySessionId").WithName(sessionId), "carve not found") } return nil, ctxerr.Wrap(ctx, err, "get carve by session ID") } diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 2a55e54482..6d8a5ef366 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -36,6 +36,10 @@ type CarveStore interface { CleanupCarves(ctx context.Context, now time.Time) (expired int, err error) } +type CarveBySessionIder interface { + CarveBySessionId(ctx context.Context, sessionId string) (*CarveMetadata, error) +} + // InstallerStore is used to communicate to a blob storage containing pre-built // fleet-osquery installers. This was originally implemented to support the // Fleet Sandbox and is not expected to be used outside of this: diff --git a/server/platform/http/errors.go b/server/platform/http/errors.go index b59785ae53..75c0cda66f 100644 --- a/server/platform/http/errors.go +++ b/server/platform/http/errors.go @@ -94,7 +94,7 @@ type PayloadTooLargeError struct { } func (e PayloadTooLargeError) Error() string { - return fmt.Sprintf("Request exceeds the max size limit of %s", units.HumanSize(float64(e.MaxRequestSize))) + return fmt.Sprintf("Request exceeds the max size limit of %s. Configure the limit: https://fleetdm.com/docs/configuration/fleet-server-configuration#server-default-max-request-body-size", units.HumanSize(float64(e.MaxRequestSize))) } func (e PayloadTooLargeError) Internal() string { diff --git a/server/service/carves.go b/server/service/carves.go index 56b9b0cc87..c204e0d738 100644 --- a/server/service/carves.go +++ b/server/service/carves.go @@ -2,10 +2,16 @@ package service import ( "context" + "encoding/base64" "errors" "fmt" + "io" + "net/http" + "strconv" + "strings" "time" + carvestorectx "github.com/fleetdm/fleet/v4/server/contexts/carvestore" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" hostctx "github.com/fleetdm/fleet/v4/server/contexts/host" "github.com/fleetdm/fleet/v4/server/contexts/logging" @@ -212,6 +218,7 @@ func (svc *Service) CarveBegin(ctx context.Context, payload fleet.CarveBeginPayl return nil, newOsqueryError("carve_size does not match block_size and block_count") } + // sessionId generated here is overriden if the carve store is S3 (in svc.carveStore.NewCarve). sessionId, err := uuid.NewRandom() if err != nil { return nil, newOsqueryError("internal error: generate session ID for carve: " + err.Error()) @@ -256,6 +263,162 @@ type carveBlockResponse struct { func (r carveBlockResponse) Error() error { return r.Err } +// DecodeRequest for the /api/v1/osquery/carve/block endpoint performs raw JSON parsing +// to prevent DoS attacks on this unauthenticated endpoint. +// Carve block requests are authenticated by their "session_id" and "request_id". +// The osquery API sends the "session_id" and "request_id" in the JSON object in the body that +// also includes the "data" field with the actual "block". If Fleet parses the full JSON to extract +// the "session_id" and "request_id" then attackers could DoS Fleet by sending big JSON documents. +// To prevent such an attack, we rely on the stability of the osquery carve endpoints (they have been +// stable for many years) and parse the body field by field. The "session_id" and "request_id" always +// come before the "data" field; thus Fleet will extract "session_id" and "request_id", perform authentication +// and if the credentials are valid parse and decode the "data" field. +func (r carveBlockRequest) DecodeRequest(ctx context.Context, req *http.Request) (any, error) { + carveStore := carvestorectx.FromContext(ctx) + if carveStore == nil { + return nil, ctxerr.New(ctx, "missing carve store from context") + } + + newAuthRequiredError := func(err error) error { + // We don't want to return details to clients. + return ctxerr.Wrap(ctx, fleet.NewAuthFailedError(err.Error()), "authentication error") + } + + readUntil := func(maxToRead int, endChar byte) (string, error) { + var s strings.Builder + endCharFound := false + for i := 0; i <= maxToRead; i++ { + character := make([]byte, 1) + if _, err := req.Body.Read(character); err != nil { + return "", fmt.Errorf("failed to read character: %w", err) + } + if character[0] == endChar { + endCharFound = true + break + } + s.Write(character) + } + if !endCharFound { + return "", fmt.Errorf(`end character not found: %q`, s.String()) + } + return s.String(), nil + } + + // 1. Must start with { + delimiter := make([]byte, 1) + if _, err := req.Body.Read(delimiter); err != nil { + return nil, newAuthRequiredError(fmt.Errorf("failed to read object start: %w", err)) + } + if string(delimiter) != "{" { + return nil, newAuthRequiredError(fmt.Errorf("expected '{', got %q", string(delimiter))) + } + // 2. Must continue with "block_id":. + blockIDKey := make([]byte, 11) + if _, err := req.Body.Read(blockIDKey); err != nil { + return nil, newAuthRequiredError(fmt.Errorf(`failed to read "block_id" key: %w`, err)) + } + if string(blockIDKey) != `"block_id":` { + return nil, newAuthRequiredError(fmt.Errorf(`expected "block_id":, got %q`, string(blockIDKey))) + } + // 3. Must continue with a number. + const maxNumberOfDigits = 19 + blockIDStr, err := readUntil(maxNumberOfDigits, ',') + if err != nil { + return nil, newAuthRequiredError(fmt.Errorf(`invalid "block_id" field: %w`, err)) + } + blockID, err := strconv.ParseInt(blockIDStr, 10, 64) + if err != nil { + return nil, newAuthRequiredError(fmt.Errorf(`invalid "block_id" format: %w`, err)) + } + // 4. Must continue with "session_id":". + sessionIDKey := make([]byte, 14) + if _, err := req.Body.Read(sessionIDKey); err != nil { + return nil, newAuthRequiredError(fmt.Errorf(`failed to read "session_id" key: %w`, err)) + } + if string(sessionIDKey) != `"session_id":"` { + return nil, newAuthRequiredError(fmt.Errorf(`expected "session_id":", got %q`, string(sessionIDKey))) + } + // 5. Must continue with a string (up to 255 chars). + const maxSizeSessionID = 255 // defined in DB + sessionID, err := readUntil(maxSizeSessionID, '"') + if err != nil { + return nil, newAuthRequiredError(fmt.Errorf(`invalid "session_id" field: %w`, err)) + } + if sessionID == "" { + return nil, newAuthRequiredError(errors.New("empty session_id")) + } + // 6. Must continue with ,"request_id":". + requestIDKey := make([]byte, 15) + if _, err := req.Body.Read(requestIDKey); err != nil { + return nil, newAuthRequiredError(fmt.Errorf(`failed to read "request_id" key: %w`, err)) + } + if string(requestIDKey) != `,"request_id":"` { + return nil, newAuthRequiredError(fmt.Errorf(`expected ,"request_id":", got %q`, string(requestIDKey))) + } + // 7. Must continue with a string (up to 64 chars). + const maxSizeRequestID = 64 // defined in DB. + requestID, err := readUntil(maxSizeRequestID, '"') + if err != nil { + return nil, newAuthRequiredError(fmt.Errorf(`invalid "request_id" field: %w`, err)) + } + if requestID == "" { + return nil, newAuthRequiredError(errors.New("empty request_id")) + } + + // + // 8. Perform authentication before continuing with the read and parse of the "data" field. + // + + carve, err := carveStore.CarveBySessionId(ctx, sessionID) + if err != nil { + return nil, newAuthRequiredError(fmt.Errorf("carve by session ID: %w", err)) + } + if requestID != carve.RequestId { + return nil, newAuthRequiredError(errors.New("request_id does not match session")) + } + + // + // 9. At this point the request is authenticated. + // + + // Must continue with ,"data":". + dataKey := make([]byte, 9) + if _, err := req.Body.Read(dataKey); err != nil { + return nil, ctxerr.Wrap(ctx, err, `failed to read "data" key`) + } + if string(dataKey) != `,"data":"` { + return nil, ctxerr.New(ctx, fmt.Sprintf(`expected ,"data":", got %s`, dataKey)) + } + + // 10. Must continue with a string with the base64 encoded data. + encodedData, err := io.ReadAll(req.Body) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, `read "data" field`) + } + if len(encodedData) < 2 { + return nil, ctxerr.New(ctx, `invalid "data" ending length`) + } + if ending := string(encodedData[len(encodedData)-2:]); ending != `"}` { + return nil, ctxerr.New(ctx, fmt.Sprintf(`invalid "data" ending: %s`, ending)) + } + // 11. Skip ending `"}` + encodedData = encodedData[:len(encodedData)-2] + // 12. Decode the base64-encoded field. + data := make([]byte, base64.RawStdEncoding.DecodedLen(len(encodedData))) + n, err := base64.StdEncoding.Decode(data, encodedData) + if err != nil { + return nil, ctxerr.Wrap(ctx, err, "base64 decode block data") + } + data = data[:n] + + return &carveBlockRequest{ + BlockId: blockID, + SessionId: sessionID, + RequestId: requestID, + Data: data, + }, nil +} + func carveBlockEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (fleet.Errorer, error) { req := request.(*carveBlockRequest) diff --git a/server/service/carves_test.go b/server/service/carves_test.go index e02c71cb8f..6c3c0a02bd 100644 --- a/server/service/carves_test.go +++ b/server/service/carves_test.go @@ -1,12 +1,19 @@ package service import ( + "bytes" "context" "errors" + "fmt" + "io" + "net/http" + strconv "strconv" + "strings" "testing" "time" "github.com/fleetdm/fleet/v4/server/authz" + carvestorectx "github.com/fleetdm/fleet/v4/server/contexts/carvestore" hostctx "github.com/fleetdm/fleet/v4/server/contexts/host" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/mock" @@ -624,3 +631,598 @@ func TestCarveCarveBlock(t *testing.T) { require.NoError(t, err) assert.True(t, ms.NewBlockFuncInvoked) } + +// MockCarveStore for testing +type mockCarveStore struct { + carves map[string]*fleet.CarveMetadata + err error +} + +func (m *mockCarveStore) CarveBySessionId(ctx context.Context, sessionID string) (*fleet.CarveMetadata, error) { + if m.err != nil { + return nil, m.err + } + c, ok := m.carves[sessionID] + if !ok { + return nil, errors.New("carve not found") + } + return c, nil +} + +func TestCarveBlockDecodeRequest(t *testing.T) { + tests := []struct { + name string + body string + ctxSetup func(ctx context.Context) context.Context + wantErr bool + wantErrType string // e.g., "AuthFailedError", "ctxerr", "" + wantErrMessage string + wantResult *carveBlockRequest + }{ + { + name: "valid request MySQL session IDs", + body: `{"block_id":123,"session_id":"23bbbcf6-6b8a-4f3a-9924-bdd084f31097","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + "23bbbcf6-6b8a-4f3a-9924-bdd084f31097": {RequestId: "req123"}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: false, + wantResult: &carveBlockRequest{ + BlockId: 123, + SessionId: "23bbbcf6-6b8a-4f3a-9924-bdd084f31097", + RequestId: "req123", + Data: []byte("database64"), + }, + }, + { + name: "valid request AWS like session IDs", + body: `{"block_id":123,"session_id":"JUMHLnWZ.A7y5ns2jUODzG8eTr5m9lvFKDD3nBN.hJ8mwr2szW0iUSNrusaE41__.wrtsNokzejFLQyNJTTqY_QN1grwAT0yXGi8A77Kf9ZJlvSiWggmncDAhVev4QXxx2PyN_GtTRPC71WGKPN2YxBFfWBjZlCZBXmPCtc4zrQ","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + "JUMHLnWZ.A7y5ns2jUODzG8eTr5m9lvFKDD3nBN.hJ8mwr2szW0iUSNrusaE41__.wrtsNokzejFLQyNJTTqY_QN1grwAT0yXGi8A77Kf9ZJlvSiWggmncDAhVev4QXxx2PyN_GtTRPC71WGKPN2YxBFfWBjZlCZBXmPCtc4zrQ": {RequestId: "req123"}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: false, + wantResult: &carveBlockRequest{ + BlockId: 123, + SessionId: "JUMHLnWZ.A7y5ns2jUODzG8eTr5m9lvFKDD3nBN.hJ8mwr2szW0iUSNrusaE41__.wrtsNokzejFLQyNJTTqY_QN1grwAT0yXGi8A77Kf9ZJlvSiWggmncDAhVev4QXxx2PyN_GtTRPC71WGKPN2YxBFfWBjZlCZBXmPCtc4zrQ", + RequestId: "req123", + Data: []byte("database64"), + }, + }, + { + name: "valid request rustfs like session IDs", + body: `{"block_id":123,"session_id":"ZGVhZDYwYTctZTVlOC00MzE1LWFhOWMtZDIzMzc5MTI4NGUyLjUzM2MxZjhhLTFiODktNDQ1YS04NTE0LTBjMWE0NDVlNjkwMXgxNzcwMTQ1Njc5NDgyNDg3NzE3","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + "ZGVhZDYwYTctZTVlOC00MzE1LWFhOWMtZDIzMzc5MTI4NGUyLjUzM2MxZjhhLTFiODktNDQ1YS04NTE0LTBjMWE0NDVlNjkwMXgxNzcwMTQ1Njc5NDgyNDg3NzE3": {RequestId: "req123"}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: false, + wantResult: &carveBlockRequest{ + BlockId: 123, + SessionId: "ZGVhZDYwYTctZTVlOC00MzE1LWFhOWMtZDIzMzc5MTI4NGUyLjUzM2MxZjhhLTFiODktNDQ1YS04NTE0LTBjMWE0NDVlNjkwMXgxNzcwMTQ1Njc5NDgyNDg3NzE3", + RequestId: "req123", + Data: []byte("database64"), + }, + }, + { + name: "valid max-sized session_id", + body: fmt.Sprintf(`{"block_id":123,"session_id":"%s","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, strings.Repeat("F", 255)), + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + strings.Repeat("F", 255): {RequestId: "req123"}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: false, + wantResult: &carveBlockRequest{ + BlockId: 123, + SessionId: strings.Repeat("F", 255), + RequestId: "req123", + Data: []byte("database64"), + }, + }, + { + name: "missing carve store", + body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { return ctx }, + wantErr: true, + wantErrMessage: "missing carve store from context", + }, + { + name: "invalid start delimiter", + body: `["block_id":123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "Authentication failed", + }, + { + name: "short non-ending session_id", + body: `{"block_id":123,"session_id":"sess123`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "Authentication failed", + }, + { + name: "max non-ending session_id", + body: fmt.Sprintf(`{"block_id":123,"session_id":"%s`, strings.Repeat("F", 256)), + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "Authentication failed", + }, + { + name: "invalid block_id key", + body: `{"blockid":123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: `expected "block_id":, got "blockid":`, + }, + { + name: "non-ending block_id key", + body: `{"block_id`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "end character not found", + }, + { + name: "invalid block_id too long", + body: `{"block_id":12345678901234567890,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "end character not found", + }, + { + name: "invalid block_id not number", + body: `{"block_id":"abc","session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "invalid \"block_id\" format", + }, + { + name: "missing session_id key", + body: `{"block_id":123,"request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: `expected "session_id":", got "request_id":"`, + }, + { + name: "missing request_id key", + body: `{"block_id":123,"session_id":"sess123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: `expected "session_id":", got "data":"ZGF0YW`, + }, + { + name: "invalid session_id key", + body: `{"block_id":123,"sess_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: `expected "session_id":", got "sess_id":"`, + }, + { + name: "invalid session_id empty", + body: `{"block_id":123,"session_id":"","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "empty session_id", + }, + { + name: "invalid session_id too long", + body: `{"block_id":123,"session_id":"` + strings.Repeat("a", 256) + `","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "end character not found", + }, + { + name: "missing session_id key, terminated body", + body: `{"block_id":123,`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "end character not found", + }, + { + name: "invalid request_id key", + body: `{"block_id":123,"session_id":"sess123","req_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: `expected ,"request_id":", got ,"req_id":"`, + }, + { + name: "invalid request_id empty", + body: `{"block_id":123,"session_id":"sess123","request_id":"","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "empty request_id", + }, + { + name: "invalid request_id too long", + body: `{"block_id":123,"session_id":"sess123","request_id":"` + strings.Repeat("F", 65) + `","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "end character not found", + }, + { + name: "max non-ending request_id", + body: fmt.Sprintf(`{"block_id":123,"session_id":"sess123","request_id":"%s`, strings.Repeat("F", 65)), + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "Authentication failed", + }, + { + name: "missing request_id key, terminated body", + body: `{"block_id":123,"session_id":"sess123"`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "Authentication failed", + }, + { + name: "valid max-sized request_id", + body: fmt.Sprintf(`{"block_id":123,"session_id":"sess123","request_id":"%s","data":"ZGF0YWJhc2U2NA=="}`, strings.Repeat("F", 64)), + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + "sess123": {RequestId: strings.Repeat("F", 64)}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: false, + wantResult: &carveBlockRequest{ + BlockId: 123, + SessionId: "sess123", + RequestId: strings.Repeat("F", 64), + Data: []byte("database64"), + }, + }, + { + name: "auth failure carve not found", + body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{}, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "carve by session ID: carve not found", + }, + { + name: "auth failure request_id mismatch", + body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + "sess123": {RequestId: "wrongreq"}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "request_id does not match session", + }, + { + name: "auth failure store error", + body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + err: errors.New("store error"), + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "carve by session ID: store error", + }, + { + name: "invalid data key", + body: `{"block_id":123,"session_id":"sess123","request_id":"req123","datum":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + "sess123": {RequestId: "req123"}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: true, + wantErrMessage: `expected ,"data":", got ,"datum":`, + }, + { + name: "missing data key", + body: `{"block_id":123,"session_id":"sess123","request_id":"req123"}`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + "sess123": {RequestId: "req123"}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: true, + wantErrMessage: `expected ,"data":", got }`, + }, + { + name: "missing data key, terminated body", + body: `{"block_id":123,"session_id":"sess123","request_id":"req123"`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + "sess123": {RequestId: "req123"}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: true, + wantErrMessage: `failed to read "data" key`, + }, + { + name: "missing data value, terminated body (ending length=0)", + body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + "sess123": {RequestId: "req123"}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: true, + wantErrMessage: `invalid "data" ending length`, + }, + { + name: "missing data value, terminated body (ending length=1)", + body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"a`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + "sess123": {RequestId: "req123"}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: true, + wantErrMessage: `invalid "data" ending length`, + }, + { + name: "empty data key", // empty block is a valid block. + body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":""}`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + "sess123": {RequestId: "req123"}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: false, + }, + { + name: "invalid data not base64", + body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"notbase64!!"}`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + "sess123": {RequestId: "req123"}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: true, + wantErrMessage: "base64 decode block data: illegal base64 data", + }, + { + name: "invalid ending", + body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + "sess123": {RequestId: "req123"}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: true, + wantErrMessage: `invalid "data" ending: ="`, + }, + { + name: "short body - after request_id", + body: `{"block_id":123,"session_id":"sess123","request_id":"req123"`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + "sess123": {RequestId: "req123"}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: true, + wantErrMessage: `failed to read "data" key: EOF`, + }, + { + name: "empty body", + body: ``, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "failed to read object start: EOF", + }, + { + name: "empty JSON", + body: `{}`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: `expected "block_id":, got }`, + }, + { + name: "unending JSON", + body: `{`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: `expected "block_id":, got }`, + }, + { + name: "string is a valid JSON", + body: `"foobar"`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "failed to read object start: EOF", + }, + { + name: "max block_id digits", + body: `{"block_id":` + strconv.FormatInt(1<<63-1, 10) + `,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + store := &mockCarveStore{ + carves: map[string]*fleet.CarveMetadata{ + "sess123": {RequestId: "req123"}, + }, + } + return carvestorectx.NewContext(ctx, store) + }, + wantErr: false, + wantResult: &carveBlockRequest{ + BlockId: 1<<63 - 1, + SessionId: "sess123", + RequestId: "req123", + Data: []byte("database64"), + }, + }, + { + name: "negative block_id", + body: `{"block_id":-123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, + ctxSetup: func(ctx context.Context) context.Context { + return carvestorectx.NewContext(ctx, &mockCarveStore{}) + }, + wantErr: true, + wantErrType: "AuthFailedError", + wantErrMessage: "invalid \"block_id\" format", + }, + // Add more edge cases as needed, e.g., special characters in strings, zero block_id, etc. + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + if tt.ctxSetup != nil { + ctx = tt.ctxSetup(ctx) + } + req := &http.Request{ + Body: io.NopCloser(bytes.NewReader([]byte(tt.body))), + } + var r carveBlockRequest + result, err := r.DecodeRequest(ctx, req) + if (err != nil) != tt.wantErr { + t.Errorf("DecodeRequest() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil { + // Check error type and message + if tt.wantErrType == "AuthFailedError" { + var afe *fleet.AuthFailedError + require.ErrorAs(t, err, &afe) + } else if tt.wantErrMessage != "" && !strings.Contains(err.Error(), tt.wantErrMessage) { + t.Errorf("error message = %v, want containing %s", err, tt.wantErrMessage) + } + return + } + got, ok := result.(*carveBlockRequest) + if !ok { + t.Errorf("result not *carveBlockRequest") + return + } + if tt.wantResult != nil { + if got.BlockId != tt.wantResult.BlockId { + t.Errorf("BlockId = %d, want %d", got.BlockId, tt.wantResult.BlockId) + } + if got.SessionId != tt.wantResult.SessionId { + t.Errorf("SessionId = %s, want %s", got.SessionId, tt.wantResult.SessionId) + } + if got.RequestId != tt.wantResult.RequestId { + t.Errorf("RequestId = %s, want %s", got.RequestId, tt.wantResult.RequestId) + } + if !bytes.Equal(got.Data, tt.wantResult.Data) { + t.Errorf("Data = %v, want %v", got.Data, tt.wantResult.Data) + } + } + }) + } +} diff --git a/server/service/handler.go b/server/service/handler.go index 9230cd5a9c..9816d0aff5 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -11,16 +11,15 @@ import ( "strings" "time" - "github.com/klauspost/compress/gzhttp" - - "github.com/docker/go-units" eeservice "github.com/fleetdm/fleet/v4/ee/server/service" "github.com/fleetdm/fleet/v4/server/config" + carvestorectx "github.com/fleetdm/fleet/v4/server/contexts/carvestore" "github.com/fleetdm/fleet/v4/server/contexts/publicip" "github.com/fleetdm/fleet/v4/server/datastore/redis" "github.com/fleetdm/fleet/v4/server/fleet" apple_mdm "github.com/fleetdm/fleet/v4/server/mdm/apple" mdmcrypto "github.com/fleetdm/fleet/v4/server/mdm/crypto" + microsoft_mdm "github.com/fleetdm/fleet/v4/server/mdm/microsoft" "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/cryptoutil" httpmdm "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/http/mdm" nanomdm_service "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/service" @@ -37,18 +36,18 @@ import ( "github.com/fleetdm/fleet/v4/server/service/middleware/mdmconfigured" "github.com/fleetdm/fleet/v4/server/service/middleware/otel" + "github.com/docker/go-units" kithttp "github.com/go-kit/kit/transport/http" kitlog "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/gorilla/mux" + "github.com/klauspost/compress/gzhttp" nanomdm_log "github.com/micromdm/nanolib/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/throttled/throttled/v2" "go.elastic.co/apm/module/apmgorilla/v2" otmiddleware "go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux" - - microsoft_mdm "github.com/fleetdm/fleet/v4/server/mdm/microsoft" ) func checkLicenseExpiration(svc fleet.Service) func(context.Context, http.ResponseWriter) context.Context { @@ -93,6 +92,13 @@ func WithHTTPSigVerifier(m mux.MiddlewareFunc) ExtraHandlerOption { } } +func setCarveStoreInRequestContext(carveStore fleet.CarveStore) kithttp.RequestFunc { + return func(ctx context.Context, r *http.Request) context.Context { + ctx = carvestorectx.NewContext(ctx, carveStore) + return ctx + } +} + // MakeHandler creates an HTTP handler for the Fleet server endpoints. func MakeHandler( svc fleet.Service, @@ -100,6 +106,7 @@ func MakeHandler( logger kitlog.Logger, limitStore throttled.GCRAStore, redisPool fleet.RedisPool, + carveStore fleet.CarveStore, featureRoutes []endpointer.HandlerRoutesFunc, extra ...ExtraHandlerOption, ) http.Handler { @@ -118,6 +125,7 @@ func MakeHandler( kithttp.ServerBefore( kithttp.PopulateRequestContext, // populate the request context with common fields auth.SetRequestsContexts(svc), + setCarveStoreInRequestContext(carveStore), ), kithttp.ServerErrorHandler(&endpointer.ErrorHandler{Logger: logger}), kithttp.ServerErrorEncoder(fleetErrorEncoder), diff --git a/server/service/handler_test.go b/server/service/handler_test.go index bc1d1ac362..a50b4ad581 100644 --- a/server/service/handler_test.go +++ b/server/service/handler_test.go @@ -31,7 +31,7 @@ func TestAPIRoutesConflicts(t *testing.T) { svc, _ := newTestService(t, ds, nil, nil) limitStore, _ := memstore.New(0) cfg := config.TestConfig() - h := MakeHandler(svc, cfg, kitlog.NewNopLogger(), limitStore, nil, nil) + h := MakeHandler(svc, cfg, kitlog.NewNopLogger(), limitStore, nil, nil, nil) router := h.(*mux.Router) type testCase struct { @@ -85,7 +85,7 @@ func TestAPIRoutesMetrics(t *testing.T) { svc, _ := newTestService(t, ds, nil, nil) limitStore, _ := memstore.New(0) - h := MakeHandler(svc, config.TestConfig(), kitlog.NewNopLogger(), limitStore, nil, nil) + h := MakeHandler(svc, config.TestConfig(), kitlog.NewNopLogger(), limitStore, nil, nil, nil) router := h.(*mux.Router) // replace all handlers with mocks, and collect the requests to make to each diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index 1a080b25fd..d30a9b5fb1 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -9234,7 +9234,7 @@ func (s *integrationTestSuite) TestCarve() { SessionId: sid + "zz", RequestId: "??", Data: []byte("p1."), - }, http.StatusNotFound, &blockResp) + }, http.StatusUnauthorized, &blockResp) // sending a block with valid session id but invalid request id s.DoJSON("POST", "/api/osquery/carve/block", carveBlockRequest{ @@ -9242,7 +9242,7 @@ func (s *integrationTestSuite) TestCarve() { SessionId: sid, RequestId: "??", Data: []byte("p1."), - }, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406 + }, http.StatusUnauthorized, &blockResp) checkCarveError := func(id uint, err string) { var getResp getCarveResponse @@ -9319,6 +9319,61 @@ func (s *integrationTestSuite) TestCarve() { checkCarveError(1, "block_id exceeds expected max (2): 3") } +func (s *integrationTestSuite) TestCarveUnauthenticated() { + t := s.T() + + verifyAuthError := func(t *testing.T, res *http.Response) { + var errs validationErrResp + err := json.NewDecoder(res.Body).Decode(&errs) + require.NoError(t, err) + res.Body.Close() + assert.Equal(t, "Authentication failed", errs.Message) + require.Len(t, errs.Errors, 1) + assert.Equal(t, "Authentication failed", errs.Errors[0].Reason) + } + + // Sending invalid format for data on purpose on purpose to check that the error is a HTTP 401 error + // vs a decoding/parsing error (this way we check it never gets to parse "data"). + for _, tc := range []struct { + testName string + rawJSONRequest string + }{ + { + testName: "empty-json", + rawJSONRequest: `{}`, + }, + { + testName: "with-spaces", // osquery does not send spaces in the JSON + rawJSONRequest: `{ + "block_id": 1, + "request_id": "invalid", + "data": 9999999999 + }`, + }, + { + testName: "without-session-id", + rawJSONRequest: `{"block_id":1,"request_id":"invalid","data":9999999999}`, + }, + { + testName: "invalid-session-id-format", + rawJSONRequest: `{"block_id":1,"session_id":2,"request_id": "invalid","data":9999999999}`, + }, + { + testName: "invalid-session-id", + rawJSONRequest: `{"block_id":1,"session_id":"invalid","request_id":"invalid","data":9999999999}`, + }, + { + testName: "invalid-JSON", + rawJSONRequest: `{"block_ASDASDASDASDASDASDASDASDASDASDASDASDASD":1}`, + }, + } { + t.Run(tc.testName, func(t *testing.T) { + res := s.DoRaw("POST", "/api/osquery/carve/block", []byte(tc.rawJSONRequest), http.StatusUnauthorized) + verifyAuthError(t, res) + }) + } +} + func (s *integrationTestSuite) TestLogLoginAttempts() { t := s.T() @@ -15646,6 +15701,3 @@ func (s *integrationTestSuite) TestDeleteCertificateTemplateSpec() { require.Equal(t, fleet.MDMOperationTypeRemove, profile.OperationType, "%s profile operation_type should be remove after deletion", tc.hostName) } } - -func (s *integrationTestSuite) TestDevieStatusMappingForHostsEndpoints() { -} diff --git a/server/service/testing_utils.go b/server/service/testing_utils.go index e100336636..2b22be40aa 100644 --- a/server/service/testing_utils.go +++ b/server/service/testing_utils.go @@ -567,7 +567,8 @@ func RunServerForTestsWithServiceWithDS(t *testing.T, ctx context.Context, ds fl require.NoError(t, condaccess.RegisterSCEP(ctx, rootMux, opts[0].ConditionalAccess.SCEPStorage, ds, logger, &cfg)) require.NoError(t, condaccess.RegisterIdP(rootMux, ds, logger, &cfg)) } - apiHandler := MakeHandler(svc, cfg, logger, limitStore, redisPool, featureRoutes, extra...) + var carveStore fleet.CarveStore = ds // In tests, we use MySQL as storage for carves. + apiHandler := MakeHandler(svc, cfg, logger, limitStore, redisPool, carveStore, featureRoutes, extra...) rootMux.Handle("/api/", apiHandler) var errHandler *errorstore.Handler ctxErrHandler := ctxerr.FromContext(ctx)