Files
Victor Lyuboslavsky 59a673bc15 Added trace sampler to use OTEL in prod. (#46595)
<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #44652 

Docs: https://github.com/fleetdm/fleet/pull/46631

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

- [x] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements), JS
inline code is prevented especially for url redirects, and untrusted
data interpolated into shell scripts/commands is validated against shell
metacharacters.

## Testing

- [x] Added/updated automated tests
- [x] QA'd all new/changed functionality manually

## Database migrations

- [x] Ensured the correct collation is explicitly set for character
columns (`COLLATE utf8mb4_unicode_ci`).

## New Fleet configuration settings

- [x] Setting(s) is/are explicitly excluded from GitOps

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Route-aware OpenTelemetry trace sampling with tiered default ratios
(very low for select high-volume routes, reduced rate for admin reads,
full sampling otherwise).
* Admin-only GET/PATCH /debug/trace_sampler to view and update sampling
ratios and a runtime "force full" toggle.
* Liveness probe endpoints (/healthz, /version, /metrics) are excluded
from tracing; settings propagate to replicas at runtime without restart.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
2026-06-02 18:53:00 -05:00

106 lines
3.6 KiB
Go

package tracing
import (
"fmt"
"sync/atomic"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
)
// Default sampling ratios. These match the seeded trace_sampler_settings row so a freshly started server uses the same ratios
// as one that has polled the DB.
const (
DefaultHighVolumeRatio = 0.001
DefaultStandardRatio = 0.02
)
// RouteTierSampler implements sdktrace.Sampler. The configured ratios live in an atomic.Pointer so SettingsPoller can swap them
// under a hot reader without locking. Tier classification is delegated to the Registry passed at construction time.
type RouteTierSampler struct {
state atomic.Pointer[samplerState]
registry *Registry
}
type samplerState struct {
highVolume sdktrace.Sampler
standard sdktrace.Sampler
always sdktrace.Sampler
never sdktrace.Sampler
forceFull bool
}
// NewRouteTierSampler returns a sampler initialized with the default ratios and force_full=false. The poller (if running)
// overwrites these on the first tick once it reads the DB row. The registry is consulted on every ShouldSample call. Routes
// added to the registry after construction are picked up immediately.
func NewRouteTierSampler(registry *Registry) *RouteTierSampler {
s := &RouteTierSampler{registry: registry}
s.state.Store(buildState(DefaultHighVolumeRatio, DefaultStandardRatio, false))
return s
}
// Apply replaces the sampler's state atomically. Out of range ratios are clamped to [0, 1] as a defensive backstop. The DB
// CHECK constraints and the PATCH handler validation reject these earlier in the pipeline.
func (s *RouteTierSampler) Apply(highVolume, standard float64, forceFull bool) {
s.state.Store(buildState(clamp01(highVolume), clamp01(standard), forceFull))
}
func clamp01(v float64) float64 {
switch {
case v < 0:
return 0
case v > 1:
return 1
}
return v
}
func buildState(highVolume, standard float64, forceFull bool) *samplerState {
return &samplerState{
highVolume: sdktrace.TraceIDRatioBased(highVolume),
standard: sdktrace.TraceIDRatioBased(standard),
always: sdktrace.AlwaysSample(),
never: sdktrace.NeverSample(),
forceFull: forceFull,
}
}
// ShouldSample implements sdktrace.Sampler. TierNever wins over ForceFull. Liveness probes should never trace, even during a
// 100% debug window. Unregistered spans (cron, MDM checkin, enroll, etc.) fall to TierAlways.
func (s *RouteTierSampler) ShouldSample(p sdktrace.SamplingParameters) sdktrace.SamplingResult {
st := s.state.Load()
tier, _ := s.registry.Lookup(p.Name)
if tier == TierNever {
return st.never.ShouldSample(p)
}
if st.forceFull {
return st.always.ShouldSample(p)
}
switch tier {
case TierHighVolume:
return st.highVolume.ShouldSample(p)
case TierStandard:
return st.standard.ShouldSample(p)
case TierAlways:
return st.always.ShouldSample(p)
}
return st.always.ShouldSample(p)
}
// Description implements sdktrace.Sampler. OTel uses it for diagnostic logging. The value should describe the sampler's
// behavior unambiguously.
func (s *RouteTierSampler) Description() string {
st := s.state.Load()
return fmt.Sprintf("RouteTierSampler{highVolume=%g,standard=%g,forceFull=%t}",
samplerRatio(st.highVolume), samplerRatio(st.standard), st.forceFull)
}
// samplerRatio extracts the configured ratio from a TraceIDRatioBased sampler for description purposes only. The SDK does not
// expose the ratio directly, so we parse its description ("TraceIDRatioBased{0.001}").
func samplerRatio(s sdktrace.Sampler) float64 {
var r float64
if _, err := fmt.Sscanf(s.Description(), "TraceIDRatioBased{%f}", &r); err != nil {
return -1
}
return r
}