Files
fleet/server/service/debug_handler.go
T
Victor Lyuboslavsky 59a673bc15 Added trace sampler to use OTEL in prod. (#46595)
<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #44652 

Docs: https://github.com/fleetdm/fleet/pull/46631

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

- [x] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements), JS
inline code is prevented especially for url redirects, and untrusted
data interpolated into shell scripts/commands is validated against shell
metacharacters.

## Testing

- [x] Added/updated automated tests
- [x] QA'd all new/changed functionality manually

## Database migrations

- [x] Ensured the correct collation is explicitly set for character
columns (`COLLATE utf8mb4_unicode_ci`).

## New Fleet configuration settings

- [x] Setting(s) is/are explicitly excluded from GitOps

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Route-aware OpenTelemetry trace sampling with tiered default ratios
(very low for select high-volume routes, reduced rate for admin reads,
full sampling otherwise).
* Admin-only GET/PATCH /debug/trace_sampler to view and update sampling
ratios and a runtime "force full" toggle.
* Liveness probe endpoints (/healthz, /version, /metrics) are excluded
from tracing; settings propagate to replicas at runtime without restart.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
2026-06-02 18:53:00 -05:00

110 lines
4.0 KiB
Go

package service
import (
"context"
"encoding/json"
"errors"
"log/slog"
"net/http"
"net/http/pprof"
"github.com/fleetdm/fleet/v4/server/config"
"github.com/fleetdm/fleet/v4/server/contexts/logging"
"github.com/fleetdm/fleet/v4/server/contexts/token"
"github.com/fleetdm/fleet/v4/server/contexts/viewer"
"github.com/fleetdm/fleet/v4/server/errorstore"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/service/middleware/auth"
kithttp "github.com/go-kit/kit/transport/http"
"github.com/gorilla/mux"
)
type debugAuthenticationMiddleware struct {
service fleet.Service
}
// Authenticate the user and ensure the account is not disabled.
func (m *debugAuthenticationMiddleware) Middleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
bearer := token.FromHTTPRequest(r)
if bearer == "" {
http.Error(w, "Please authenticate", http.StatusUnauthorized)
return
}
ctx := token.NewContext(context.Background(), bearer)
v, err := auth.AuthViewer(ctx, string(bearer), m.service)
if err != nil {
http.Error(w, "Invalid authentication", http.StatusUnauthorized)
return
}
if !v.CanPerformActions() || v.User.GlobalRole == nil || *v.User.GlobalRole != fleet.RoleAdmin {
http.Error(w, "Unauthorized", http.StatusForbidden)
return
}
// Attach the authenticated viewer to the request context so downstream debug handlers can record who triggered an
// action (e.g. updating trace sampler settings).
next.ServeHTTP(w, r.WithContext(viewer.NewContext(r.Context(), *v)))
})
}
func jsonHandler(
logger *slog.Logger,
jsonGenerator func(ctx context.Context) (any, error),
) func(rw http.ResponseWriter, r *http.Request) {
return func(rw http.ResponseWriter, r *http.Request) {
lc := &logging.LoggingContext{SkipUser: true} // The debug handler does not save the logged-in user.
ctx := logging.NewContext(kithttp.PopulateRequestContext(r.Context(), r), lc)
ctx = logging.WithStartTime(ctx)
jsonData, err := jsonGenerator(ctx)
if err != nil {
lc.SetErrs(err)
lc.Log(ctx, logger)
var sce kithttp.StatusCoder
if errors.As(err, &sce) {
rw.WriteHeader(sce.StatusCode())
_, _ = rw.Write([]byte(err.Error()))
return
}
rw.WriteHeader(http.StatusInternalServerError)
return
}
b, err := json.MarshalIndent(jsonData, "", " ")
if err != nil {
lc.SetErrs(err)
lc.Log(ctx, logger)
rw.WriteHeader(http.StatusInternalServerError)
return
}
rw.Write(b) //nolint:errcheck
}
}
// MakeDebugHandler creates an HTTP handler for the Fleet debug endpoints.
func MakeDebugHandler(svc fleet.Service, config config.FleetConfig, logger *slog.Logger, eh *errorstore.Handler, ds fleet.Datastore) http.Handler {
r := mux.NewRouter()
r.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
r.HandleFunc("/debug/pprof/profile", pprof.Profile)
r.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
r.HandleFunc("/debug/pprof/trace", pprof.Trace)
r.Handle("/debug/errors", eh)
r.PathPrefix("/debug/pprof/").HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { pprof.Index(rw, req) })
r.HandleFunc("/debug/migrations", jsonHandler(logger, func(ctx context.Context) (interface{}, error) { return ds.MigrationStatus(ctx) }))
r.HandleFunc("/debug/db/locks", jsonHandler(logger, func(ctx context.Context) (interface{}, error) { return ds.DBLocks(ctx) }))
r.HandleFunc("/debug/db/innodb-status", jsonHandler(logger, func(ctx context.Context) (interface{}, error) { return ds.InnoDBStatus(ctx) }))
r.HandleFunc("/debug/db/process-list", jsonHandler(logger, func(ctx context.Context) (interface{}, error) { return ds.ProcessList(ctx) }))
r.HandleFunc("/debug/trace_sampler", jsonHandler(logger, func(ctx context.Context) (any, error) {
return ds.GetTraceSamplerSettings(ctx)
})).Methods(http.MethodGet)
r.HandleFunc("/debug/trace_sampler", patchTraceSamplerHandler(logger, ds)).Methods(http.MethodPatch)
mw := &debugAuthenticationMiddleware{
service: svc,
}
r.Use(mw.Middleware)
return r
}