Authenticate carve block endpoint before parsing the "data" field (#39353)

- [X] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

- [X] If paths of existing endpoints are modified without backwards
compatibility, checked the frontend/CLI for any necessary changes

## Testing

- [X] Added/updated automated tests
- [X] QA'd all new/changed functionality manually

---------

Co-authored-by: Magnus Jensen <magnus@fleetdm.com>

server/contexts/carvestore/carvestore.go

@@ -0,0 +1,23 @@
+package carvestore
+
+import (
+	"context"
+
+	"github.com/fleetdm/fleet/v4/server/fleet"
+)
+
+type key int
+
+const carveStoreKey key = 0
+
+func NewContext(ctx context.Context, svc fleet.CarveBySessionIder) context.Context {
+	return context.WithValue(ctx, carveStoreKey, svc)
+}
+
+func FromContext(ctx context.Context) fleet.CarveBySessionIder {
+	svc, ok := ctx.Value(carveStoreKey).(fleet.CarveBySessionIder)
+	if !ok {
+		return nil
+	}
+	return svc
+}

server/datastore/mysql/carves.go

@@ -198,7 +198,7 @@ func (ds *Datastore) CarveBySessionId(ctx context.Context, sessionId string) (*f
 	var metadata fleet.CarveMetadata
 	if err := sqlx.GetContext(ctx, ds.reader(ctx), &metadata, stmt, sessionId); err != nil {
 		if err == sql.ErrNoRows {
-			return nil, ctxerr.Wrap(ctx, notFound("CarveBySessionId").WithName(sessionId))
+			return nil, ctxerr.Wrap(ctx, notFound("CarveBySessionId").WithName(sessionId), "carve not found")
 		}
 		return nil, ctxerr.Wrap(ctx, err, "get carve by session ID")
 	}

server/fleet/datastore.go

@@ -36,6 +36,10 @@ type CarveStore interface {
 	CleanupCarves(ctx context.Context, now time.Time) (expired int, err error)
 }
 
+type CarveBySessionIder interface {
+	CarveBySessionId(ctx context.Context, sessionId string) (*CarveMetadata, error)
+}
+
 // InstallerStore is used to communicate to a blob storage containing pre-built
 // fleet-osquery installers. This was originally implemented to support the
 // Fleet Sandbox and is not expected to be used outside of this:

server/platform/http/errors.go

@@ -94,7 +94,7 @@ type PayloadTooLargeError struct {
 }
 
 func (e PayloadTooLargeError) Error() string {
-	return fmt.Sprintf("Request exceeds the max size limit of %s", units.HumanSize(float64(e.MaxRequestSize)))
+	return fmt.Sprintf("Request exceeds the max size limit of %s. Configure the limit: https://fleetdm.com/docs/configuration/fleet-server-configuration#server-default-max-request-body-size", units.HumanSize(float64(e.MaxRequestSize)))
 }
 
 func (e PayloadTooLargeError) Internal() string {

server/service/carves.go

@@ -2,10 +2,16 @@ package service
 
 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
 	"time"
 
+	carvestorectx "github.com/fleetdm/fleet/v4/server/contexts/carvestore"
 	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
 	hostctx "github.com/fleetdm/fleet/v4/server/contexts/host"
 	"github.com/fleetdm/fleet/v4/server/contexts/logging"
@@ -212,6 +218,7 @@ func (svc *Service) CarveBegin(ctx context.Context, payload fleet.CarveBeginPayl
 		return nil, newOsqueryError("carve_size does not match block_size and block_count")
 	}
 
+	// sessionId generated here is overriden if the carve store is S3 (in svc.carveStore.NewCarve).
 	sessionId, err := uuid.NewRandom()
 	if err != nil {
 		return nil, newOsqueryError("internal error: generate session ID for carve: " + err.Error())
@@ -256,6 +263,162 @@ type carveBlockResponse struct {
 
 func (r carveBlockResponse) Error() error { return r.Err }
 
+// DecodeRequest for the /api/v1/osquery/carve/block endpoint performs raw JSON parsing
+// to prevent DoS attacks on this unauthenticated endpoint.
+// Carve block requests are authenticated by their "session_id" and "request_id".
+// The osquery API sends the "session_id" and "request_id" in the JSON object in the body that
+// also includes the "data" field with the actual "block". If Fleet parses the full JSON to extract
+// the "session_id" and "request_id" then attackers could DoS Fleet by sending big JSON documents.
+// To prevent such an attack, we rely on the stability of the osquery carve endpoints (they have been
+// stable for many years) and parse the body field by field. The "session_id" and "request_id" always
+// come before the "data" field; thus Fleet will extract "session_id" and "request_id", perform authentication
+// and if the credentials are valid parse and decode the "data" field.
+func (r carveBlockRequest) DecodeRequest(ctx context.Context, req *http.Request) (any, error) {
+	carveStore := carvestorectx.FromContext(ctx)
+	if carveStore == nil {
+		return nil, ctxerr.New(ctx, "missing carve store from context")
+	}
+
+	newAuthRequiredError := func(err error) error {
+		// We don't want to return details to clients.
+		return ctxerr.Wrap(ctx, fleet.NewAuthFailedError(err.Error()), "authentication error")
+	}
+
+	readUntil := func(maxToRead int, endChar byte) (string, error) {
+		var s strings.Builder
+		endCharFound := false
+		for i := 0; i <= maxToRead; i++ {
+			character := make([]byte, 1)
+			if _, err := req.Body.Read(character); err != nil {
+				return "", fmt.Errorf("failed to read character: %w", err)
+			}
+			if character[0] == endChar {
+				endCharFound = true
+				break
+			}
+			s.Write(character)
+		}
+		if !endCharFound {
+			return "", fmt.Errorf(`end character not found: %q`, s.String())
+		}
+		return s.String(), nil
+	}
+
+	// 1. Must start with {
+	delimiter := make([]byte, 1)
+	if _, err := req.Body.Read(delimiter); err != nil {
+		return nil, newAuthRequiredError(fmt.Errorf("failed to read object start: %w", err))
+	}
+	if string(delimiter) != "{" {
+		return nil, newAuthRequiredError(fmt.Errorf("expected '{', got %q", string(delimiter)))
+	}
+	// 2. Must continue with "block_id":.
+	blockIDKey := make([]byte, 11)
+	if _, err := req.Body.Read(blockIDKey); err != nil {
+		return nil, newAuthRequiredError(fmt.Errorf(`failed to read "block_id" key: %w`, err))
+	}
+	if string(blockIDKey) != `"block_id":` {
+		return nil, newAuthRequiredError(fmt.Errorf(`expected "block_id":, got %q`, string(blockIDKey)))
+	}
+	// 3. Must continue with a number.
+	const maxNumberOfDigits = 19
+	blockIDStr, err := readUntil(maxNumberOfDigits, ',')
+	if err != nil {
+		return nil, newAuthRequiredError(fmt.Errorf(`invalid "block_id" field: %w`, err))
+	}
+	blockID, err := strconv.ParseInt(blockIDStr, 10, 64)
+	if err != nil {
+		return nil, newAuthRequiredError(fmt.Errorf(`invalid "block_id" format: %w`, err))
+	}
+	// 4. Must continue with "session_id":".
+	sessionIDKey := make([]byte, 14)
+	if _, err := req.Body.Read(sessionIDKey); err != nil {
+		return nil, newAuthRequiredError(fmt.Errorf(`failed to read "session_id" key: %w`, err))
+	}
+	if string(sessionIDKey) != `"session_id":"` {
+		return nil, newAuthRequiredError(fmt.Errorf(`expected "session_id":", got %q`, string(sessionIDKey)))
+	}
+	// 5. Must continue with a string (up to 255 chars).
+	const maxSizeSessionID = 255 // defined in DB
+	sessionID, err := readUntil(maxSizeSessionID, '"')
+	if err != nil {
+		return nil, newAuthRequiredError(fmt.Errorf(`invalid "session_id" field: %w`, err))
+	}
+	if sessionID == "" {
+		return nil, newAuthRequiredError(errors.New("empty session_id"))
+	}
+	// 6. Must continue with ,"request_id":".
+	requestIDKey := make([]byte, 15)
+	if _, err := req.Body.Read(requestIDKey); err != nil {
+		return nil, newAuthRequiredError(fmt.Errorf(`failed to read "request_id" key: %w`, err))
+	}
+	if string(requestIDKey) != `,"request_id":"` {
+		return nil, newAuthRequiredError(fmt.Errorf(`expected ,"request_id":", got %q`, string(requestIDKey)))
+	}
+	// 7. Must continue with a string (up to 64 chars).
+	const maxSizeRequestID = 64 // defined in DB.
+	requestID, err := readUntil(maxSizeRequestID, '"')
+	if err != nil {
+		return nil, newAuthRequiredError(fmt.Errorf(`invalid "request_id" field: %w`, err))
+	}
+	if requestID == "" {
+		return nil, newAuthRequiredError(errors.New("empty request_id"))
+	}
+
+	//
+	// 8. Perform authentication before continuing with the read and parse of the "data" field.
+	//
+
+	carve, err := carveStore.CarveBySessionId(ctx, sessionID)
+	if err != nil {
+		return nil, newAuthRequiredError(fmt.Errorf("carve by session ID: %w", err))
+	}
+	if requestID != carve.RequestId {
+		return nil, newAuthRequiredError(errors.New("request_id does not match session"))
+	}
+
+	//
+	// 9. At this point the request is authenticated.
+	//
+
+	// Must continue with ,"data":".
+	dataKey := make([]byte, 9)
+	if _, err := req.Body.Read(dataKey); err != nil {
+		return nil, ctxerr.Wrap(ctx, err, `failed to read "data" key`)
+	}
+	if string(dataKey) != `,"data":"` {
+		return nil, ctxerr.New(ctx, fmt.Sprintf(`expected ,"data":", got %s`, dataKey))
+	}
+
+	// 10. Must continue with a string with the base64 encoded data.
+	encodedData, err := io.ReadAll(req.Body)
+	if err != nil {
+		return nil, ctxerr.Wrap(ctx, err, `read "data" field`)
+	}
+	if len(encodedData) < 2 {
+		return nil, ctxerr.New(ctx, `invalid "data" ending length`)
+	}
+	if ending := string(encodedData[len(encodedData)-2:]); ending != `"}` {
+		return nil, ctxerr.New(ctx, fmt.Sprintf(`invalid "data" ending: %s`, ending))
+	}
+	// 11. Skip ending `"}`
+	encodedData = encodedData[:len(encodedData)-2]
+	// 12. Decode the base64-encoded field.
+	data := make([]byte, base64.RawStdEncoding.DecodedLen(len(encodedData)))
+	n, err := base64.StdEncoding.Decode(data, encodedData)
+	if err != nil {
+		return nil, ctxerr.Wrap(ctx, err, "base64 decode block data")
+	}
+	data = data[:n]
+
+	return &carveBlockRequest{
+		BlockId:   blockID,
+		SessionId: sessionID,
+		RequestId: requestID,
+		Data:      data,
+	}, nil
+}
+
 func carveBlockEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (fleet.Errorer, error) {
 	req := request.(*carveBlockRequest)
 

server/service/carves_test.go

@@ -1,12 +1,19 @@
 package service
 
 import (
+	"bytes"
 	"context"
 	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	strconv "strconv"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/fleetdm/fleet/v4/server/authz"
+	carvestorectx "github.com/fleetdm/fleet/v4/server/contexts/carvestore"
 	hostctx "github.com/fleetdm/fleet/v4/server/contexts/host"
 	"github.com/fleetdm/fleet/v4/server/fleet"
 	"github.com/fleetdm/fleet/v4/server/mock"
@@ -624,3 +631,598 @@ func TestCarveCarveBlock(t *testing.T) {
 	require.NoError(t, err)
 	assert.True(t, ms.NewBlockFuncInvoked)
 }
+
+// MockCarveStore for testing
+type mockCarveStore struct {
+	carves map[string]*fleet.CarveMetadata
+	err    error
+}
+
+func (m *mockCarveStore) CarveBySessionId(ctx context.Context, sessionID string) (*fleet.CarveMetadata, error) {
+	if m.err != nil {
+		return nil, m.err
+	}
+	c, ok := m.carves[sessionID]
+	if !ok {
+		return nil, errors.New("carve not found")
+	}
+	return c, nil
+}
+
+func TestCarveBlockDecodeRequest(t *testing.T) {
+	tests := []struct {
+		name           string
+		body           string
+		ctxSetup       func(ctx context.Context) context.Context
+		wantErr        bool
+		wantErrType    string // e.g., "AuthFailedError", "ctxerr", ""
+		wantErrMessage string
+		wantResult     *carveBlockRequest
+	}{
+		{
+			name: "valid request MySQL session IDs",
+			body: `{"block_id":123,"session_id":"23bbbcf6-6b8a-4f3a-9924-bdd084f31097","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						"23bbbcf6-6b8a-4f3a-9924-bdd084f31097": {RequestId: "req123"},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr: false,
+			wantResult: &carveBlockRequest{
+				BlockId:   123,
+				SessionId: "23bbbcf6-6b8a-4f3a-9924-bdd084f31097",
+				RequestId: "req123",
+				Data:      []byte("database64"),
+			},
+		},
+		{
+			name: "valid request AWS like session IDs",
+			body: `{"block_id":123,"session_id":"JUMHLnWZ.A7y5ns2jUODzG8eTr5m9lvFKDD3nBN.hJ8mwr2szW0iUSNrusaE41__.wrtsNokzejFLQyNJTTqY_QN1grwAT0yXGi8A77Kf9ZJlvSiWggmncDAhVev4QXxx2PyN_GtTRPC71WGKPN2YxBFfWBjZlCZBXmPCtc4zrQ","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						"JUMHLnWZ.A7y5ns2jUODzG8eTr5m9lvFKDD3nBN.hJ8mwr2szW0iUSNrusaE41__.wrtsNokzejFLQyNJTTqY_QN1grwAT0yXGi8A77Kf9ZJlvSiWggmncDAhVev4QXxx2PyN_GtTRPC71WGKPN2YxBFfWBjZlCZBXmPCtc4zrQ": {RequestId: "req123"},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr: false,
+			wantResult: &carveBlockRequest{
+				BlockId:   123,
+				SessionId: "JUMHLnWZ.A7y5ns2jUODzG8eTr5m9lvFKDD3nBN.hJ8mwr2szW0iUSNrusaE41__.wrtsNokzejFLQyNJTTqY_QN1grwAT0yXGi8A77Kf9ZJlvSiWggmncDAhVev4QXxx2PyN_GtTRPC71WGKPN2YxBFfWBjZlCZBXmPCtc4zrQ",
+				RequestId: "req123",
+				Data:      []byte("database64"),
+			},
+		},
+		{
+			name: "valid request rustfs like session IDs",
+			body: `{"block_id":123,"session_id":"ZGVhZDYwYTctZTVlOC00MzE1LWFhOWMtZDIzMzc5MTI4NGUyLjUzM2MxZjhhLTFiODktNDQ1YS04NTE0LTBjMWE0NDVlNjkwMXgxNzcwMTQ1Njc5NDgyNDg3NzE3","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						"ZGVhZDYwYTctZTVlOC00MzE1LWFhOWMtZDIzMzc5MTI4NGUyLjUzM2MxZjhhLTFiODktNDQ1YS04NTE0LTBjMWE0NDVlNjkwMXgxNzcwMTQ1Njc5NDgyNDg3NzE3": {RequestId: "req123"},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr: false,
+			wantResult: &carveBlockRequest{
+				BlockId:   123,
+				SessionId: "ZGVhZDYwYTctZTVlOC00MzE1LWFhOWMtZDIzMzc5MTI4NGUyLjUzM2MxZjhhLTFiODktNDQ1YS04NTE0LTBjMWE0NDVlNjkwMXgxNzcwMTQ1Njc5NDgyNDg3NzE3",
+				RequestId: "req123",
+				Data:      []byte("database64"),
+			},
+		},
+		{
+			name: "valid max-sized session_id",
+			body: fmt.Sprintf(`{"block_id":123,"session_id":"%s","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`, strings.Repeat("F", 255)),
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						strings.Repeat("F", 255): {RequestId: "req123"},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr: false,
+			wantResult: &carveBlockRequest{
+				BlockId:   123,
+				SessionId: strings.Repeat("F", 255),
+				RequestId: "req123",
+				Data:      []byte("database64"),
+			},
+		},
+		{
+			name:           "missing carve store",
+			body:           `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup:       func(ctx context.Context) context.Context { return ctx },
+			wantErr:        true,
+			wantErrMessage: "missing carve store from context",
+		},
+		{
+			name: "invalid start delimiter",
+			body: `["block_id":123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "Authentication failed",
+		},
+		{
+			name: "short non-ending session_id",
+			body: `{"block_id":123,"session_id":"sess123`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "Authentication failed",
+		},
+		{
+			name: "max non-ending session_id",
+			body: fmt.Sprintf(`{"block_id":123,"session_id":"%s`, strings.Repeat("F", 256)),
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "Authentication failed",
+		},
+		{
+			name: "invalid block_id key",
+			body: `{"blockid":123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: `expected "block_id":, got "blockid":`,
+		},
+		{
+			name: "non-ending block_id key",
+			body: `{"block_id`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "end character not found",
+		},
+		{
+			name: "invalid block_id too long",
+			body: `{"block_id":12345678901234567890,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "end character not found",
+		},
+		{
+			name: "invalid block_id not number",
+			body: `{"block_id":"abc","session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "invalid \"block_id\" format",
+		},
+		{
+			name: "missing session_id key",
+			body: `{"block_id":123,"request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: `expected "session_id":", got "request_id":"`,
+		},
+		{
+			name: "missing request_id key",
+			body: `{"block_id":123,"session_id":"sess123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: `expected "session_id":", got "data":"ZGF0YW`,
+		},
+		{
+			name: "invalid session_id key",
+			body: `{"block_id":123,"sess_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: `expected "session_id":", got "sess_id":"`,
+		},
+		{
+			name: "invalid session_id empty",
+			body: `{"block_id":123,"session_id":"","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "empty session_id",
+		},
+		{
+			name: "invalid session_id too long",
+			body: `{"block_id":123,"session_id":"` + strings.Repeat("a", 256) + `","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "end character not found",
+		},
+		{
+			name: "missing session_id key, terminated body",
+			body: `{"block_id":123,`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "end character not found",
+		},
+		{
+			name: "invalid request_id key",
+			body: `{"block_id":123,"session_id":"sess123","req_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: `expected ,"request_id":", got ,"req_id":"`,
+		},
+		{
+			name: "invalid request_id empty",
+			body: `{"block_id":123,"session_id":"sess123","request_id":"","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "empty request_id",
+		},
+		{
+			name: "invalid request_id too long",
+			body: `{"block_id":123,"session_id":"sess123","request_id":"` + strings.Repeat("F", 65) + `","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "end character not found",
+		},
+		{
+			name: "max non-ending request_id",
+			body: fmt.Sprintf(`{"block_id":123,"session_id":"sess123","request_id":"%s`, strings.Repeat("F", 65)),
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "Authentication failed",
+		},
+		{
+			name: "missing request_id key, terminated body",
+			body: `{"block_id":123,"session_id":"sess123"`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "Authentication failed",
+		},
+		{
+			name: "valid max-sized request_id",
+			body: fmt.Sprintf(`{"block_id":123,"session_id":"sess123","request_id":"%s","data":"ZGF0YWJhc2U2NA=="}`, strings.Repeat("F", 64)),
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						"sess123": {RequestId: strings.Repeat("F", 64)},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr: false,
+			wantResult: &carveBlockRequest{
+				BlockId:   123,
+				SessionId: "sess123",
+				RequestId: strings.Repeat("F", 64),
+				Data:      []byte("database64"),
+			},
+		},
+		{
+			name: "auth failure carve not found",
+			body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "carve by session ID: carve not found",
+		},
+		{
+			name: "auth failure request_id mismatch",
+			body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						"sess123": {RequestId: "wrongreq"},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "request_id does not match session",
+		},
+		{
+			name: "auth failure store error",
+			body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					err: errors.New("store error"),
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "carve by session ID: store error",
+		},
+		{
+			name: "invalid data key",
+			body: `{"block_id":123,"session_id":"sess123","request_id":"req123","datum":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						"sess123": {RequestId: "req123"},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr:        true,
+			wantErrMessage: `expected ,"data":", got ,"datum":`,
+		},
+		{
+			name: "missing data key",
+			body: `{"block_id":123,"session_id":"sess123","request_id":"req123"}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						"sess123": {RequestId: "req123"},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr:        true,
+			wantErrMessage: `expected ,"data":", got }`,
+		},
+		{
+			name: "missing data key, terminated body",
+			body: `{"block_id":123,"session_id":"sess123","request_id":"req123"`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						"sess123": {RequestId: "req123"},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr:        true,
+			wantErrMessage: `failed to read "data" key`,
+		},
+		{
+			name: "missing data value, terminated body (ending length=0)",
+			body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						"sess123": {RequestId: "req123"},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr:        true,
+			wantErrMessage: `invalid "data" ending length`,
+		},
+		{
+			name: "missing data value, terminated body (ending length=1)",
+			body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"a`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						"sess123": {RequestId: "req123"},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr:        true,
+			wantErrMessage: `invalid "data" ending length`,
+		},
+		{
+			name: "empty data key", // empty block is a valid block.
+			body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":""}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						"sess123": {RequestId: "req123"},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr: false,
+		},
+		{
+			name: "invalid data not base64",
+			body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"notbase64!!"}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						"sess123": {RequestId: "req123"},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr:        true,
+			wantErrMessage: "base64 decode block data: illegal base64 data",
+		},
+		{
+			name: "invalid ending",
+			body: `{"block_id":123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						"sess123": {RequestId: "req123"},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr:        true,
+			wantErrMessage: `invalid "data" ending: ="`,
+		},
+		{
+			name: "short body - after request_id",
+			body: `{"block_id":123,"session_id":"sess123","request_id":"req123"`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						"sess123": {RequestId: "req123"},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr:        true,
+			wantErrMessage: `failed to read "data" key: EOF`,
+		},
+		{
+			name: "empty body",
+			body: ``,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "failed to read object start: EOF",
+		},
+		{
+			name: "empty JSON",
+			body: `{}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: `expected "block_id":, got }`,
+		},
+		{
+			name: "unending JSON",
+			body: `{`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: `expected "block_id":, got }`,
+		},
+		{
+			name: "string is a valid JSON",
+			body: `"foobar"`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "failed to read object start: EOF",
+		},
+		{
+			name: "max block_id digits",
+			body: `{"block_id":` + strconv.FormatInt(1<<63-1, 10) + `,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				store := &mockCarveStore{
+					carves: map[string]*fleet.CarveMetadata{
+						"sess123": {RequestId: "req123"},
+					},
+				}
+				return carvestorectx.NewContext(ctx, store)
+			},
+			wantErr: false,
+			wantResult: &carveBlockRequest{
+				BlockId:   1<<63 - 1,
+				SessionId: "sess123",
+				RequestId: "req123",
+				Data:      []byte("database64"),
+			},
+		},
+		{
+			name: "negative block_id",
+			body: `{"block_id":-123,"session_id":"sess123","request_id":"req123","data":"ZGF0YWJhc2U2NA=="}`,
+			ctxSetup: func(ctx context.Context) context.Context {
+				return carvestorectx.NewContext(ctx, &mockCarveStore{})
+			},
+			wantErr:        true,
+			wantErrType:    "AuthFailedError",
+			wantErrMessage: "invalid \"block_id\" format",
+		},
+		// Add more edge cases as needed, e.g., special characters in strings, zero block_id, etc.
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
+			if tt.ctxSetup != nil {
+				ctx = tt.ctxSetup(ctx)
+			}
+			req := &http.Request{
+				Body: io.NopCloser(bytes.NewReader([]byte(tt.body))),
+			}
+			var r carveBlockRequest
+			result, err := r.DecodeRequest(ctx, req)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("DecodeRequest() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if err != nil {
+				// Check error type and message
+				if tt.wantErrType == "AuthFailedError" {
+					var afe *fleet.AuthFailedError
+					require.ErrorAs(t, err, &afe)
+				} else if tt.wantErrMessage != "" && !strings.Contains(err.Error(), tt.wantErrMessage) {
+					t.Errorf("error message = %v, want containing %s", err, tt.wantErrMessage)
+				}
+				return
+			}
+			got, ok := result.(*carveBlockRequest)
+			if !ok {
+				t.Errorf("result not *carveBlockRequest")
+				return
+			}
+			if tt.wantResult != nil {
+				if got.BlockId != tt.wantResult.BlockId {
+					t.Errorf("BlockId = %d, want %d", got.BlockId, tt.wantResult.BlockId)
+				}
+				if got.SessionId != tt.wantResult.SessionId {
+					t.Errorf("SessionId = %s, want %s", got.SessionId, tt.wantResult.SessionId)
+				}
+				if got.RequestId != tt.wantResult.RequestId {
+					t.Errorf("RequestId = %s, want %s", got.RequestId, tt.wantResult.RequestId)
+				}
+				if !bytes.Equal(got.Data, tt.wantResult.Data) {
+					t.Errorf("Data = %v, want %v", got.Data, tt.wantResult.Data)
+				}
+			}
+		})
+	}
+}

server/service/handler.go

@@ -11,16 +11,15 @@ import (
 	"strings"
 	"time"
 
-	"github.com/klauspost/compress/gzhttp"
-
-	"github.com/docker/go-units"
 	eeservice "github.com/fleetdm/fleet/v4/ee/server/service"
 	"github.com/fleetdm/fleet/v4/server/config"
+	carvestorectx "github.com/fleetdm/fleet/v4/server/contexts/carvestore"
 	"github.com/fleetdm/fleet/v4/server/contexts/publicip"
 	"github.com/fleetdm/fleet/v4/server/datastore/redis"
 	"github.com/fleetdm/fleet/v4/server/fleet"
 	apple_mdm "github.com/fleetdm/fleet/v4/server/mdm/apple"
 	mdmcrypto "github.com/fleetdm/fleet/v4/server/mdm/crypto"
+	microsoft_mdm "github.com/fleetdm/fleet/v4/server/mdm/microsoft"
 	"github.com/fleetdm/fleet/v4/server/mdm/nanomdm/cryptoutil"
 	httpmdm "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/http/mdm"
 	nanomdm_service "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/service"
@@ -37,18 +36,18 @@ import (
 	"github.com/fleetdm/fleet/v4/server/service/middleware/mdmconfigured"
 	"github.com/fleetdm/fleet/v4/server/service/middleware/otel"
 
+	"github.com/docker/go-units"
 	kithttp "github.com/go-kit/kit/transport/http"
 	kitlog "github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/gorilla/mux"
+	"github.com/klauspost/compress/gzhttp"
 	nanomdm_log "github.com/micromdm/nanolib/log"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/throttled/throttled/v2"
 	"go.elastic.co/apm/module/apmgorilla/v2"
 	otmiddleware "go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux"
-
-	microsoft_mdm "github.com/fleetdm/fleet/v4/server/mdm/microsoft"
 )
 
 func checkLicenseExpiration(svc fleet.Service) func(context.Context, http.ResponseWriter) context.Context {
@@ -93,6 +92,13 @@ func WithHTTPSigVerifier(m mux.MiddlewareFunc) ExtraHandlerOption {
 	}
 }
 
+func setCarveStoreInRequestContext(carveStore fleet.CarveStore) kithttp.RequestFunc {
+	return func(ctx context.Context, r *http.Request) context.Context {
+		ctx = carvestorectx.NewContext(ctx, carveStore)
+		return ctx
+	}
+}
+
 // MakeHandler creates an HTTP handler for the Fleet server endpoints.
 func MakeHandler(
 	svc fleet.Service,
@@ -100,6 +106,7 @@ func MakeHandler(
 	logger kitlog.Logger,
 	limitStore throttled.GCRAStore,
 	redisPool fleet.RedisPool,
+	carveStore fleet.CarveStore,
 	featureRoutes []endpointer.HandlerRoutesFunc,
 	extra ...ExtraHandlerOption,
 ) http.Handler {
@@ -118,6 +125,7 @@ func MakeHandler(
 		kithttp.ServerBefore(
 			kithttp.PopulateRequestContext, // populate the request context with common fields
 			auth.SetRequestsContexts(svc),
+			setCarveStoreInRequestContext(carveStore),
 		),
 		kithttp.ServerErrorHandler(&endpointer.ErrorHandler{Logger: logger}),
 		kithttp.ServerErrorEncoder(fleetErrorEncoder),

server/service/handler_test.go

@@ -31,7 +31,7 @@ func TestAPIRoutesConflicts(t *testing.T) {
 	svc, _ := newTestService(t, ds, nil, nil)
 	limitStore, _ := memstore.New(0)
 	cfg := config.TestConfig()
-	h := MakeHandler(svc, cfg, kitlog.NewNopLogger(), limitStore, nil, nil)
+	h := MakeHandler(svc, cfg, kitlog.NewNopLogger(), limitStore, nil, nil, nil)
 	router := h.(*mux.Router)
 
 	type testCase struct {
@@ -85,7 +85,7 @@ func TestAPIRoutesMetrics(t *testing.T) {
 
 	svc, _ := newTestService(t, ds, nil, nil)
 	limitStore, _ := memstore.New(0)
-	h := MakeHandler(svc, config.TestConfig(), kitlog.NewNopLogger(), limitStore, nil, nil)
+	h := MakeHandler(svc, config.TestConfig(), kitlog.NewNopLogger(), limitStore, nil, nil, nil)
 	router := h.(*mux.Router)
 
 	// replace all handlers with mocks, and collect the requests to make to each

server/service/integration_core_test.go

@@ -9234,7 +9234,7 @@ func (s *integrationTestSuite) TestCarve() {
 		SessionId: sid + "zz",
 		RequestId: "??",
 		Data:      []byte("p1."),
-	}, http.StatusNotFound, &blockResp)
+	}, http.StatusUnauthorized, &blockResp)
 
 	// sending a block with valid session id but invalid request id
 	s.DoJSON("POST", "/api/osquery/carve/block", carveBlockRequest{
@@ -9242,7 +9242,7 @@ func (s *integrationTestSuite) TestCarve() {
 		SessionId: sid,
 		RequestId: "??",
 		Data:      []byte("p1."),
-	}, http.StatusInternalServerError, &blockResp) // TODO: should be 400, see #4406
+	}, http.StatusUnauthorized, &blockResp)
 
 	checkCarveError := func(id uint, err string) {
 		var getResp getCarveResponse
@@ -9319,6 +9319,61 @@ func (s *integrationTestSuite) TestCarve() {
 	checkCarveError(1, "block_id exceeds expected max (2): 3")
 }
 
+func (s *integrationTestSuite) TestCarveUnauthenticated() {
+	t := s.T()
+
+	verifyAuthError := func(t *testing.T, res *http.Response) {
+		var errs validationErrResp
+		err := json.NewDecoder(res.Body).Decode(&errs)
+		require.NoError(t, err)
+		res.Body.Close()
+		assert.Equal(t, "Authentication failed", errs.Message)
+		require.Len(t, errs.Errors, 1)
+		assert.Equal(t, "Authentication failed", errs.Errors[0].Reason)
+	}
+
+	// Sending invalid format for data on purpose on purpose to check that the error is a HTTP 401 error
+	// vs a decoding/parsing error (this way we check it never gets to parse "data").
+	for _, tc := range []struct {
+		testName       string
+		rawJSONRequest string
+	}{
+		{
+			testName:       "empty-json",
+			rawJSONRequest: `{}`,
+		},
+		{
+			testName: "with-spaces", // osquery does not send spaces in the JSON
+			rawJSONRequest: `{
+				"block_id":   1,
+				"request_id": "invalid",
+				"data":      9999999999
+			}`,
+		},
+		{
+			testName:       "without-session-id",
+			rawJSONRequest: `{"block_id":1,"request_id":"invalid","data":9999999999}`,
+		},
+		{
+			testName:       "invalid-session-id-format",
+			rawJSONRequest: `{"block_id":1,"session_id":2,"request_id": "invalid","data":9999999999}`,
+		},
+		{
+			testName:       "invalid-session-id",
+			rawJSONRequest: `{"block_id":1,"session_id":"invalid","request_id":"invalid","data":9999999999}`,
+		},
+		{
+			testName:       "invalid-JSON",
+			rawJSONRequest: `{"block_ASDASDASDASDASDASDASDASDASDASDASDASDASD":1}`,
+		},
+	} {
+		t.Run(tc.testName, func(t *testing.T) {
+			res := s.DoRaw("POST", "/api/osquery/carve/block", []byte(tc.rawJSONRequest), http.StatusUnauthorized)
+			verifyAuthError(t, res)
+		})
+	}
+}
+
 func (s *integrationTestSuite) TestLogLoginAttempts() {
 	t := s.T()
 
@@ -15646,6 +15701,3 @@ func (s *integrationTestSuite) TestDeleteCertificateTemplateSpec() {
 		require.Equal(t, fleet.MDMOperationTypeRemove, profile.OperationType, "%s profile operation_type should be remove after deletion", tc.hostName)
 	}
 }
-
-func (s *integrationTestSuite) TestDevieStatusMappingForHostsEndpoints() {
-}

server/service/testing_utils.go

@@ -567,7 +567,8 @@ func RunServerForTestsWithServiceWithDS(t *testing.T, ctx context.Context, ds fl
 		require.NoError(t, condaccess.RegisterSCEP(ctx, rootMux, opts[0].ConditionalAccess.SCEPStorage, ds, logger, &cfg))
 		require.NoError(t, condaccess.RegisterIdP(rootMux, ds, logger, &cfg))
 	}
-	apiHandler := MakeHandler(svc, cfg, logger, limitStore, redisPool, featureRoutes, extra...)
+	var carveStore fleet.CarveStore = ds // In tests, we use MySQL as storage for carves.
+	apiHandler := MakeHandler(svc, cfg, logger, limitStore, redisPool, carveStore, featureRoutes, extra...)
 	rootMux.Handle("/api/", apiHandler)
 	var errHandler *errorstore.Handler
 	ctxErrHandler := ctxerr.FromContext(ctx)