Files
fleet/server/service/debug_trace_sampler.go
T
Victor Lyuboslavsky 59a673bc15 Added trace sampler to use OTEL in prod. (#46595)
<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #44652 

Docs: https://github.com/fleetdm/fleet/pull/46631

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

- [x] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements), JS
inline code is prevented especially for url redirects, and untrusted
data interpolated into shell scripts/commands is validated against shell
metacharacters.

## Testing

- [x] Added/updated automated tests
- [x] QA'd all new/changed functionality manually

## Database migrations

- [x] Ensured the correct collation is explicitly set for character
columns (`COLLATE utf8mb4_unicode_ci`).

## New Fleet configuration settings

- [x] Setting(s) is/are explicitly excluded from GitOps

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Route-aware OpenTelemetry trace sampling with tiered default ratios
(very low for select high-volume routes, reduced rate for admin reads,
full sampling otherwise).
* Admin-only GET/PATCH /debug/trace_sampler to view and update sampling
ratios and a runtime "force full" toggle.
* Liveness probe endpoints (/healthz, /version, /metrics) are excluded
from tracing; settings propagate to replicas at runtime without restart.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
2026-06-02 18:53:00 -05:00

114 lines
4.7 KiB
Go

package service
import (
"encoding/json"
"errors"
"fmt"
"log/slog"
"net/http"
"time"
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
"github.com/fleetdm/fleet/v4/server/contexts/viewer"
"github.com/fleetdm/fleet/v4/server/fleet"
)
// traceSamplerPatchRequest is the PATCH payload. Fields are pointers so we can distinguish "unset" from a zero value. PATCH
// semantics mean only the provided fields are applied.
type traceSamplerPatchRequest struct {
HighVolumeRatio *float64 `json:"high_volume_ratio,omitempty"`
StandardRatio *float64 `json:"standard_ratio,omitempty"`
ForceFull *bool `json:"force_full,omitempty"`
}
// patchTraceSamplerHandler returns the PATCH /debug/trace_sampler handler. The GET path is wired separately in
// MakeDebugHandler via the existing jsonHandler helper, matching the convention used by /debug/migrations and /debug/db/*.
//
// PATCH is necessarily bespoke because no other /debug/ endpoint takes a request body. It validates ratios in [0, 1] and
// persists the change. The replica's in memory sampler picks up the change on the next poller tick (default 60s).
func patchTraceSamplerHandler(logger *slog.Logger, ds fleet.Datastore) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
// The /debug auth middleware always installs the viewer in context. If it is missing here, the middleware was bypassed
// and we should refuse to record a change rather than silently log user_id=0. That value is indistinguishable from a
// real user id of 0 and weakens the audit trail.
v, ok := viewer.FromContext(r.Context())
if !ok {
handleServerError(w, r, logger, "debug trace_sampler PATCH refused: viewer missing from context", "viewer required",
errors.New("viewer missing from context"))
return
}
var req traceSamplerPatchRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, fmt.Sprintf("invalid JSON body: %v", err), http.StatusBadRequest)
return
}
if err := validateTraceSamplerPatch(req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
current, err := ds.GetTraceSamplerSettings(r.Context())
if err != nil {
handleServerError(w, r, logger, "debug trace_sampler PATCH read-modify failed", "internal error", err)
return
}
if req.HighVolumeRatio != nil {
current.HighVolumeRatio = *req.HighVolumeRatio
}
if req.StandardRatio != nil {
current.StandardRatio = *req.StandardRatio
}
if req.ForceFull != nil {
current.ForceFull = *req.ForceFull
}
if err := ds.SetTraceSamplerSettings(r.Context(), current); err != nil {
handleServerError(w, r, logger, "debug trace_sampler PATCH write failed", "internal error", err)
return
}
logger.InfoContext(r.Context(), "trace sampler settings updated",
"high_volume_ratio", current.HighVolumeRatio,
"standard_ratio", current.StandardRatio,
"force_full", current.ForceFull,
"updated_by_user_id", v.UserID(),
)
// Return the updated row so callers can confirm what was applied. Drop UpdatedAt: the row was read before the write,
// so current.UpdatedAt is the pre-write timestamp (stale and confusing). omitzero on the struct tag skips the field
// when zero. Operators who want the post-write timestamp can do a follow-up GET.
current.UpdatedAt = time.Time{}
b, err := json.MarshalIndent(current, "", " ")
if err != nil {
handleServerError(w, r, logger, "debug trace_sampler PATCH encode response failed", "encoding response", err)
return
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
_, _ = w.Write(b)
}
}
// handleServerError centralizes the internal-error response used throughout the trace sampler PATCH handler: it logs logMsg
// with err, records err on the context for the error-handling middleware, and writes clientMsg to the client as a 500.
func handleServerError(w http.ResponseWriter, r *http.Request, logger *slog.Logger, logMsg, clientMsg string, err error) {
logger.ErrorContext(r.Context(), logMsg, "err", err)
ctxerr.Handle(r.Context(), err)
http.Error(w, clientMsg, http.StatusInternalServerError)
}
func validateTraceSamplerPatch(req traceSamplerPatchRequest) error {
if req.HighVolumeRatio == nil && req.StandardRatio == nil && req.ForceFull == nil {
return errors.New("request body must include at least one of high_volume_ratio, standard_ratio, force_full")
}
if req.HighVolumeRatio != nil && (*req.HighVolumeRatio < 0 || *req.HighVolumeRatio > 1) {
return fmt.Errorf("high_volume_ratio must be in [0, 1], got %v", *req.HighVolumeRatio)
}
if req.StandardRatio != nil && (*req.StandardRatio < 0 || *req.StandardRatio > 1) {
return fmt.Errorf("standard_ratio must be in [0, 1], got %v", *req.StandardRatio)
}
return nil
}