Victor Lyuboslavsky
59a673bc15
<!-- Add the related story/sub-task/bug number, like Resolves #123, or remove if NA --> **Related issue:** Resolves #44652 Docs: https://github.com/fleetdm/fleet/pull/46631 # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files) for more information. - [x] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements), JS inline code is prevented especially for url redirects, and untrusted data interpolated into shell scripts/commands is validated against shell metacharacters. ## Testing - [x] Added/updated automated tests - [x] QA'd all new/changed functionality manually ## Database migrations - [x] Ensured the correct collation is explicitly set for character columns (`COLLATE utf8mb4_unicode_ci`). ## New Fleet configuration settings - [x] Setting(s) is/are explicitly excluded from GitOps <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Route-aware OpenTelemetry trace sampling with tiered default ratios (very low for select high-volume routes, reduced rate for admin reads, full sampling otherwise). * Admin-only GET/PATCH /debug/trace_sampler to view and update sampling ratios and a runtime "force full" toggle. * Liveness probe endpoints (/healthz, /version, /metrics) are excluded from tracing; settings propagate to replicas at runtime without restart. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
73 lines
2.5 KiB
Go
73 lines
2.5 KiB
Go
package tracing
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"go.opentelemetry.io/otel"
|
|
"go.opentelemetry.io/otel/trace"
|
|
)
|
|
|
|
// settingsPollInterval is how often each Fleet replica re-reads trace_sampler_settings. 60s matches industry defaults for
|
|
// feature flag style runtime config.
|
|
const settingsPollInterval = 60 * time.Second
|
|
|
|
// pollTracer instruments the poll loop. When OTEL is not configured the global provider returns a no-op tracer, so this is
|
|
// free when tracing is disabled.
|
|
var pollTracer = otel.Tracer("github.com/fleetdm/fleet/v4/server/platform/tracing")
|
|
|
|
// settingsReader is the minimal datastore surface the poller needs. The full fleet.Datastore is large. Depending only on this
|
|
// interface keeps the poller's tests cheap and the package free of cross context coupling.
|
|
type settingsReader interface {
|
|
GetTraceSamplerSettings(ctx context.Context) (*Settings, error)
|
|
}
|
|
|
|
// StartSettingsPoller runs the polling loop until ctx is cancelled. On each tick it reads the trace_sampler_settings row,
|
|
// compares the values to the last applied state, and calls sampler. Apply only when something changed.
|
|
//
|
|
// On startup it does one immediate read so the sampler picks up the row's current values without waiting a full interval. If
|
|
// the first read fails, the sampler keeps its compile time defaults and a warning is logged. The next tick will try again.
|
|
func StartSettingsPoller(ctx context.Context, sampler *RouteTierSampler, ds settingsReader, logger *slog.Logger) {
|
|
pollAndApply := func(last *Settings) *Settings {
|
|
spanCtx, span := pollTracer.Start(ctx, "tracing.poll_settings",
|
|
trace.WithNewRoot(),
|
|
trace.WithSpanKind(trace.SpanKindInternal),
|
|
)
|
|
defer span.End()
|
|
|
|
got, err := ds.GetTraceSamplerSettings(spanCtx)
|
|
if err != nil {
|
|
logger.ErrorContext(spanCtx, "trace sampler settings poll failed", "err", err)
|
|
return last
|
|
}
|
|
if last != nil &&
|
|
got.HighVolumeRatio == last.HighVolumeRatio &&
|
|
got.StandardRatio == last.StandardRatio &&
|
|
got.ForceFull == last.ForceFull {
|
|
return last
|
|
}
|
|
sampler.Apply(got.HighVolumeRatio, got.StandardRatio, got.ForceFull)
|
|
logger.InfoContext(spanCtx, "trace sampler settings applied",
|
|
"high_volume_ratio", got.HighVolumeRatio,
|
|
"standard_ratio", got.StandardRatio,
|
|
"force_full", got.ForceFull,
|
|
)
|
|
return got
|
|
}
|
|
|
|
var last *Settings
|
|
last = pollAndApply(last)
|
|
|
|
ticker := time.NewTicker(settingsPollInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
last = pollAndApply(last)
|
|
}
|
|
}
|
|
}
|