Files
fleet/server/platform/tracing/settings_poller_test.go
Victor Lyuboslavsky 59a673bc15 Added trace sampler to use OTEL in prod. (#46595)
<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #44652 

Docs: https://github.com/fleetdm/fleet/pull/46631

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

- [x] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements), JS
inline code is prevented especially for url redirects, and untrusted
data interpolated into shell scripts/commands is validated against shell
metacharacters.

## Testing

- [x] Added/updated automated tests
- [x] QA'd all new/changed functionality manually

## Database migrations

- [x] Ensured the correct collation is explicitly set for character
columns (`COLLATE utf8mb4_unicode_ci`).

## New Fleet configuration settings

- [x] Setting(s) is/are explicitly excluded from GitOps

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Route-aware OpenTelemetry trace sampling with tiered default ratios
(very low for select high-volume routes, reduced rate for admin reads,
full sampling otherwise).
* Admin-only GET/PATCH /debug/trace_sampler to view and update sampling
ratios and a runtime "force full" toggle.
* Liveness probe endpoints (/healthz, /version, /metrics) are excluded
from tracing; settings propagate to replicas at runtime without restart.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
2026-06-02 18:53:00 -05:00

140 lines
4.1 KiB
Go

package tracing
import (
"context"
"errors"
"io"
"log/slog"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/stretchr/testify/require"
)
type stubReader struct {
mu atomic.Pointer[Settings]
err atomic.Pointer[error]
getCalls atomic.Int32
}
func (s *stubReader) set(settings Settings) {
s.mu.Store(&settings)
}
func (s *stubReader) setErr(err error) {
s.err.Store(&err)
}
func (s *stubReader) GetTraceSamplerSettings(_ context.Context) (*Settings, error) {
s.getCalls.Add(1)
if e := s.err.Load(); e != nil && *e != nil {
return nil, *e
}
if cur := s.mu.Load(); cur != nil {
out := *cur
return &out, nil
}
return &Settings{
HighVolumeRatio: DefaultHighVolumeRatio,
StandardRatio: DefaultStandardRatio,
}, nil
}
func discardLogger() *slog.Logger {
return slog.New(slog.NewTextHandler(io.Discard, nil))
}
func TestStartSettingsPoller_AppliesInitialReadImmediately(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
r := &stubReader{}
r.set(Settings{
HighVolumeRatio: 0.4,
StandardRatio: 0.8,
ForceFull: true,
})
sampler := NewRouteTierSampler(NewRegistry())
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
go StartSettingsPoller(ctx, sampler, r, discardLogger())
// Wait blocks until every other goroutine in the bubble is durably blocked. The poller does its synchronous initial
// read, applies, then blocks on the ticker. So once Wait returns, the apply has happened.
synctest.Wait()
require.Equal(t, int32(1), r.getCalls.Load(), "exactly one poll should have happened by now")
st := sampler.state.Load()
require.True(t, st.forceFull, "initial read must apply force_full")
})
}
func TestStartSettingsPoller_HandlesErrorGracefully(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
r := &stubReader{}
r.setErr(errors.New("db unavailable"))
sampler := NewRouteTierSampler(NewRegistry())
// Capture current state. It should remain unchanged after a failed poll.
beforeForceFull := sampler.state.Load().forceFull
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
go StartSettingsPoller(ctx, sampler, r, discardLogger())
synctest.Wait()
require.Equal(t, int32(1), r.getCalls.Load(), "the failed poll still counts as one read")
require.Equal(t, beforeForceFull, sampler.state.Load().forceFull,
"sampler state must be unchanged when the read fails")
})
}
func TestStartSettingsPoller_AppliesChangeOnTick(t *testing.T) {
// Locks in the actual 60s ticker behavior. The old tests could only assert the initial synchronous read because waiting a
// real minute per test was untenable. With synctest, advancing time is free.
synctest.Test(t, func(t *testing.T) {
r := &stubReader{}
r.set(Settings{
HighVolumeRatio: 0.4,
StandardRatio: 0.8,
ForceFull: true,
})
sampler := NewRouteTierSampler(NewRegistry())
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
go StartSettingsPoller(ctx, sampler, r, discardLogger())
// Initial synchronous poll completes.
synctest.Wait()
require.Equal(t, int32(1), r.getCalls.Load())
require.True(t, sampler.state.Load().forceFull)
// Flip the stub to a new value. Advance past the next ticker fire; the synthetic clock advances while everything is
// blocked, the ticker fires, the poller re-polls and applies the new state. Wait then ensures the poller has
// re-blocked before we assert.
r.set(Settings{
HighVolumeRatio: 0.001,
StandardRatio: 0.02,
ForceFull: false,
})
time.Sleep(settingsPollInterval + time.Nanosecond)
synctest.Wait()
require.Equal(t, int32(2), r.getCalls.Load(), "second poll must have fired after one ticker interval")
require.False(t, sampler.state.Load().forceFull, "ticker poll must apply the new state")
// One more tick with no underlying change should still call Get but should be a no-op for Apply. We verify by the
// invariant that the state is unchanged from the previous assertion.
time.Sleep(settingsPollInterval + time.Nanosecond)
synctest.Wait()
require.Equal(t, int32(3), r.getCalls.Load())
require.False(t, sampler.state.Load().forceFull)
})
}