Added missing OpenTelemetry instrumentation to several API endpoints. (#32960)

Fixes #32331 

Manually tested all paths. `/test` path removed in
https://github.com/fleetdm/fleet/pull/32962

Also added support for sending errors to OpenTelemetry, like we do for
APM/Sentry.

# Checklist for submitter

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

## Testing
- [x] QA'd all new/changed functionality manually


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added OpenTelemetry tracing across core HTTP endpoints (health,
version, assets, metrics, enroll/root, debug, Apple MDM, SCEP, SCIM)
with dynamic per-request route instrumentation.
* Enhanced error reporting to include OpenTelemetry spans/events with
contextual user/host attributes.

* **Tests**
* Added unit tests validating SCIM and error-handling telemetry, span
naming, and sensitive-data redaction.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

server/contexts/ctxerr/ctxerr.go

@@ -25,6 +25,9 @@ import (
 	"github.com/fleetdm/fleet/v4/server/fleet"
 	"github.com/getsentry/sentry-go"
 	"go.elastic.co/apm/v2"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 )
 
 type key int
@@ -291,20 +294,61 @@ func FromContext(ctx context.Context) Handler {
 
 // Handle handles err by passing it to the registered error handler,
 // deduplicating it and storing it for a configured duration. It also takes
-// care of sending it to the configured APM, if any.
+// care of sending it to the configured OpenTelemetry/APM/Sentry, if any.
 func Handle(ctx context.Context, err error) {
+	if err == nil {
+		return
+	}
+
 	// as a last resource, wrap the error if there isn't
 	// a FleetError in the chain
 	var ferr *FleetError
 	if !errors.As(err, &ferr) {
-		err = Wrap(ctx, err, "missing FleetError in chain")
+		wrapped := wrapError(ctx, "missing FleetError in chain", err, nil)
+		if wrapped == nil {
+			return // shouldn't happen since err is not nil, but be safe
+		}
+		// wrapError returns an error interface, but we know it's a *FleetError
+		ferr = wrapped.(*FleetError)
 	}
 
-	cause := err
-	if ferr := FleetCause(err); ferr != nil {
+	cause := ferr
+	if rootCause := FleetCause(ferr); rootCause != nil {
 		// use the FleetCause error so we send the most relevant stacktrace to APM
 		// (the one from the initial New/Wrap call).
-		cause = ferr
+		cause = rootCause
+	}
+
+	// send to OpenTelemetry if there's an active span
+	if span := trace.SpanFromContext(ctx); span != nil && span.IsRecording() {
+		// Mark the current span as failed by setting the error status.
+		// This status can be overridden if we recovered from the error.
+		span.SetStatus(codes.Error, cause.Error())
+
+		// Build attributes for the exception event
+		attrs := []attribute.KeyValue{
+			attribute.String("exception.type", fmt.Sprintf("%T", cause)),
+			attribute.String("exception.message", cause.Error()),
+			attribute.String("exception.stacktrace", strings.Join(cause.Stack(), "\n")),
+		}
+
+		// Add contextual information if available (same as Sentry)
+		v, _ := viewer.FromContext(ctx)
+		h, _ := host.FromContext(ctx)
+
+		if v.User != nil {
+			attrs = append(attrs,
+				// Not sending the email here as it may contain sensitive information (PII).
+				attribute.Int64("user.id", int64(v.User.ID)), //nolint:gosec
+			)
+		} else if h != nil {
+			attrs = append(attrs,
+				attribute.String("host.hostname", h.Hostname),
+				attribute.Int64("host.id", int64(h.ID)), //nolint:gosec
+			)
+		}
+
+		span.AddEvent("exception", trace.WithAttributes(attrs...))
 	}
 
 	// send to elastic APM
@@ -338,7 +382,7 @@ func Handle(ctx context.Context, err error) {
 	}
 
 	if eh := FromContext(ctx); eh != nil {
-		eh.Store(err)
+		eh.Store(ferr)
 	}
 }
 

server/contexts/ctxerr/ctxerr_otel_test.go

@@ -0,0 +1,129 @@
+package ctxerr
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/fleetdm/fleet/v4/server/contexts/host"
+	"github.com/fleetdm/fleet/v4/server/contexts/viewer"
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/sdk/trace/tracetest"
+)
+
+func TestHandleSendsContextToOTEL(t *testing.T) {
+	t.Parallel()
+	testCases := []struct {
+		name          string
+		setupContext  func(context.Context) context.Context
+		errorMessage  string
+		expectedAttrs map[string]any // expected attributes in the exception event
+	}{
+		{
+			name: "with user context",
+			setupContext: func(ctx context.Context) context.Context {
+				testUser := &fleet.User{
+					ID:    123,
+					Email: "test@example.com",
+				}
+				return viewer.NewContext(ctx, viewer.Viewer{User: testUser})
+			},
+			errorMessage: "test error with user context",
+			expectedAttrs: map[string]any{
+				"user.id": int64(123),
+			},
+		},
+		{
+			name: "with host context",
+			setupContext: func(ctx context.Context) context.Context {
+				testHost := &fleet.Host{
+					ID:       456,
+					Hostname: "test-host.example.com",
+				}
+				return host.NewContext(ctx, testHost)
+			},
+			errorMessage: "test error with host context",
+			expectedAttrs: map[string]any{
+				"host.hostname": "test-host.example.com",
+				"host.id":       int64(456),
+			},
+		},
+		{
+			name: "without additional context",
+			setupContext: func(ctx context.Context) context.Context {
+				return ctx // no additional context
+			},
+			errorMessage:  "test error without context",
+			expectedAttrs: map[string]any{},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Create a test span recorder and tracer provider
+			sr := tracetest.NewSpanRecorder()
+			tp := trace.NewTracerProvider(trace.WithSpanProcessor(sr))
+
+			// Create a context with an active span using the test tracer
+			tracer := tp.Tracer("test")
+			ctx, span := tracer.Start(t.Context(), "test-span")
+			defer span.End()
+
+			// Setup context with test-specific data
+			ctx = tc.setupContext(ctx)
+
+			// Create and handle an error
+			err := New(ctx, tc.errorMessage)
+			Handle(ctx, err)
+
+			// Force span to end so we can check recorded data
+			span.End()
+
+			// Check that the exception event was created
+			spans := sr.Ended()
+			require.Len(t, spans, 1)
+
+			// Find the exception event
+			events := spans[0].Events()
+			var exceptionEvent *trace.Event
+			for i := range events {
+				if events[i].Name == "exception" {
+					exceptionEvent = &events[i]
+					break
+				}
+			}
+			require.NotNil(t, exceptionEvent, "Expected to find an exception event")
+
+			// Check all expected attributes are present
+			attributes := make(map[string]any)
+			for _, attr := range exceptionEvent.Attributes {
+				switch attr.Key {
+				case "user.id", "host.id":
+					attributes[string(attr.Key)] = attr.Value.AsInt64()
+				default:
+					attributes[string(attr.Key)] = attr.Value.AsString()
+				}
+			}
+
+			// Always check for stack trace
+			stackTrace, ok := attributes["exception.stacktrace"].(string)
+			assert.True(t, ok, "Expected exception.stacktrace attribute")
+			assert.Contains(t, stackTrace, "TestHandleSendsContextToOTEL", "Stack trace should contain test function name")
+			assert.True(t, strings.Contains(stackTrace, "\n"), "Stack trace should be formatted with newlines")
+
+			// Check for exception message and type
+			assert.Equal(t, tc.errorMessage, attributes["exception.message"])
+			assert.Equal(t, "*ctxerr.FleetError", attributes["exception.type"])
+
+			// Check test-specific expected attributes
+			for expectedKey, expectedValue := range tc.expectedAttrs {
+				actualValue, found := attributes[expectedKey]
+				assert.True(t, found, "Expected to find attribute %s", expectedKey)
+				assert.Equal(t, expectedValue, actualValue, "Attribute %s should match", expectedKey)
+			}
+		})
+	}
+}