Added trace sampler to use OTEL in prod. (#46595)

<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #44652 

Docs: https://github.com/fleetdm/fleet/pull/46631

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

- [x] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements), JS
inline code is prevented especially for url redirects, and untrusted
data interpolated into shell scripts/commands is validated against shell
metacharacters.

## Testing

- [x] Added/updated automated tests
- [x] QA'd all new/changed functionality manually

## Database migrations

- [x] Ensured the correct collation is explicitly set for character
columns (`COLLATE utf8mb4_unicode_ci`).

## New Fleet configuration settings

- [x] Setting(s) is/are explicitly excluded from GitOps

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Route-aware OpenTelemetry trace sampling with tiered default ratios
(very low for select high-volume routes, reduced rate for admin reads,
full sampling otherwise).
* Admin-only GET/PATCH /debug/trace_sampler to view and update sampling
ratios and a runtime "force full" toggle.
* Liveness probe endpoints (/healthz, /version, /metrics) are excluded
from tracing; settings propagate to replicas at runtime without restart.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

changes/44652-trace-sampler

@@ -0,0 +1,2 @@
+- Added route-aware head sampling for OpenTelemetry trace export. When `tracing_enabled` is on, agent firehose endpoints (osquery distributed read/write, orbit ping/config, device desktop/ping) are sampled at 0.1% by default, admin reads at 2%, and everything else (enroll, SCEP, MDM checkin, cron jobs, GitOps batch) at 100%. Liveness probes (`/healthz`, `/version`, `/metrics`) are dropped unconditionally.
+- Added `GET`/`PATCH /debug/trace_sampler` (admin only, behind the existing `/debug` auth) for adjusting ratios or flipping a 100% `force_full` debug window at runtime. Each Fleet replica polls the new `trace_sampler_settings` row every 60 seconds and applies changes without a restart.

cmd/fleet/otel.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/fleetdm/fleet/v4/server/config"
+	"github.com/fleetdm/fleet/v4/server/platform/tracing"
 	"github.com/fleetdm/fleet/v4/server/version"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc"
@@ -17,26 +18,25 @@ import (
 	semconv "go.opentelemetry.io/otel/semconv/v1.40.0"
 )
 
-// initOTELProviders constructs the OpenTelemetry trace, metric, and (when
-// log export is enabled) log providers. Returns nil providers when OTEL is
-// disabled in the configuration. Fatal errors during exporter setup are
-// reported through initFatal so the server can fail fast at startup.
+// initOTELProviders constructs the OpenTelemetry trace, metric, and (when log export is enabled) log providers. Returns nil
+// providers when OTEL is disabled in the configuration. Fatal errors during exporter setup are reported through initFatal so
+// the server can fail fast at startup.
 //
-// As a side effect, the constructed tracer and meter providers are registered
-// as the OTEL globals via otel.SetTracerProvider and otel.SetMeterProvider —
-// matching the original inline behavior in runServeCmd.
-func initOTELProviders(cfg config.FleetConfig, initFatal func(err error, msg string)) (
+// The traceRegistry is consulted on every sampling decision. Populating it (with route to tier mappings) is the caller's
+// responsibility. The canonical pattern is for each bounded context to register its own routes at startup. See
+// server/service/tracing_tiers.go for the legacy non modularized routes.
+func initOTELProviders(cfg config.FleetConfig, traceRegistry *tracing.Registry, initFatal func(err error, msg string)) (
 	*otelsdklog.LoggerProvider,
 	*sdktrace.TracerProvider,
 	*sdkmetric.MeterProvider,
+	*tracing.RouteTierSampler,
 ) {
 	if !cfg.OTELEnabled() {
-		return nil, nil, nil
+		return nil, nil, nil, nil
 	}
 
-	// Create shared resource with service identification attributes.
-	// OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES env vars can override
-	// the defaults below.
+	// Create shared resource with service identification attributes. OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES env vars
+	// can override the defaults below.
 	res, err := resource.New(context.Background(),
 		resource.WithSchemaURL(semconv.SchemaURL),
 		resource.WithAttributes(
@@ -48,9 +48,8 @@ func initOTELProviders(cfg config.FleetConfig, initFatal func(err error, msg str
 	)
 	if err != nil {
 		initFatal(err, "Failed to create OTEL resource")
-		// Returning here makes the function safe even if a caller's
-		// initFatal does not terminate (e.g., tests using a recorder).
-		return nil, nil, nil
+		// Returning here makes the function safe even if a caller's initFatal does not terminate (e.g. tests using a recorder).
+		return nil, nil, nil, nil
 	}
 
 	// Initialize OTEL traces.
@@ -59,16 +58,34 @@ func initOTELProviders(cfg config.FleetConfig, initFatal func(err error, msg str
 	))
 	if err != nil {
 		initFatal(err, "Failed to initialize OTEL trace exporter")
-		return nil, nil, nil
+		return nil, nil, nil, nil
 	}
-	// Configure batch span processor with smaller batch size to avoid exceeding
-	// message size limits (4MB default limit).
+	// Configure batch span processor with smaller batch size to avoid exceeding message size limits (4MB default limit).
 	batchSpanProcessor := sdktrace.NewBatchSpanProcessor(otlpTraceExporter,
 		sdktrace.WithMaxExportBatchSize(256), // Reduce from default 512 to 256
 	)
+	// Route aware head sampler. The sampler starts with compile time defaults. The settings poller re-reads trace_sampler_settings
+	// every 60s and calls Apply on change. We wrap with ParentBased so a remote sampled parent (e.g. an upstream span) keeps the
+	// whole trace coherent even on the hot agent path.
+	sampler := tracing.NewRouteTierSampler(traceRegistry)
+	// ParentBased semantics:
+	//
+	//   Remote parent SAMPLED: honor the upstream decision via AlwaysSample.
+	//
+	//   Remote parent NOT SAMPLED: honor it via NeverSample, keeping trace coherence across services. Defaulting to our local
+	//   sampler here would let us locally sample a span whose upstream parent was explicitly dropped, breaking distributed
+	//   traces.
+	//
+	//   No remote parent: use our route aware sampler.
+	parentBased := sdktrace.ParentBased(
+		sampler,
+		sdktrace.WithRemoteParentSampled(sdktrace.AlwaysSample()),
+		sdktrace.WithRemoteParentNotSampled(sdktrace.NeverSample()),
+	)
 	tracerProvider := sdktrace.NewTracerProvider(
 		sdktrace.WithResource(res),
 		sdktrace.WithSpanProcessor(batchSpanProcessor),
+		sdktrace.WithSampler(parentBased),
 	)
 	otel.SetTracerProvider(tracerProvider)
 
@@ -78,7 +95,7 @@ func initOTELProviders(cfg config.FleetConfig, initFatal func(err error, msg str
 	)
 	if err != nil {
 		initFatal(err, "Failed to initialize OTEL metrics exporter")
-		return nil, nil, nil
+		return nil, nil, nil, nil
 	}
 
 	// Create views to rename otelsql metrics to match what OpenTelemetry Signoz expects.
@@ -125,7 +142,7 @@ func initOTELProviders(cfg config.FleetConfig, initFatal func(err error, msg str
 		)
 		if err != nil {
 			initFatal(err, "Failed to initialize OTEL log exporter")
-			return nil, nil, nil
+			return nil, nil, nil, nil
 		}
 		loggerProvider = otelsdklog.NewLoggerProvider(
 			otelsdklog.WithResource(res),
@@ -133,5 +150,5 @@ func initOTELProviders(cfg config.FleetConfig, initFatal func(err error, msg str
 		)
 	}
 
-	return loggerProvider, tracerProvider, meterProvider
+	return loggerProvider, tracerProvider, meterProvider, sampler
 }

cmd/fleet/otel_test.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/fleetdm/fleet/v4/server/config"
+	"github.com/fleetdm/fleet/v4/server/platform/tracing"
 	otelsdklog "go.opentelemetry.io/otel/sdk/log"
 	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
@@ -41,12 +42,13 @@ func TestInitOTELProviders_DisabledReturnsNilProviders(t *testing.T) {
 	require.False(t, cfg.OTELEnabled(), "precondition: default config must have OTEL disabled")
 
 	called := false
-	lp, tp, mp := initOTELProviders(cfg, func(err error, msg string) { called = true })
+	lp, tp, mp, sampler := initOTELProviders(cfg, tracing.NewRegistry(), func(err error, msg string) { called = true })
 
 	require.False(t, called, "initFatal must not be called when OTEL is disabled")
 	require.Nil(t, lp)
 	require.Nil(t, tp)
 	require.Nil(t, mp)
+	require.Nil(t, sampler, "sampler must be nil when OTEL is disabled")
 }
 
 func TestInitOTELProviders_EnabledReturnsTracerAndMeterProviders(t *testing.T) {
@@ -60,13 +62,14 @@ func TestInitOTELProviders_EnabledReturnsTracerAndMeterProviders(t *testing.T) {
 	require.True(t, cfg.OTELEnabled(), "precondition: tracing-enabled config must have OTEL enabled")
 
 	called := false
-	lp, tp, mp := initOTELProviders(cfg, func(err error, msg string) { called = true })
+	lp, tp, mp, sampler := initOTELProviders(cfg, tracing.NewRegistry(), func(err error, msg string) { called = true })
 	shutdownOTELProviders(t, lp, tp, mp)
 
 	require.False(t, called, "initFatal must not be called for a healthy enabled config")
 	require.Nil(t, lp, "logger provider should be nil when OtelLogsEnabled is false")
 	require.NotNil(t, tp)
 	require.NotNil(t, mp)
+	require.NotNil(t, sampler, "sampler must be returned when OTEL is enabled")
 }
 
 func TestInitOTELProviders_LogExportEnabledReturnsLoggerProvider(t *testing.T) {
@@ -75,11 +78,12 @@ func TestInitOTELProviders_LogExportEnabledReturnsLoggerProvider(t *testing.T) {
 	}
 
 	called := false
-	lp, tp, mp := initOTELProviders(cfg, func(err error, msg string) { called = true })
+	lp, tp, mp, sampler := initOTELProviders(cfg, tracing.NewRegistry(), func(err error, msg string) { called = true })
 	shutdownOTELProviders(t, lp, tp, mp)
 
 	require.False(t, called)
 	require.NotNil(t, lp, "logger provider should be set when OtelLogsEnabled is true")
 	require.NotNil(t, tp)
 	require.NotNil(t, mp)
+	require.NotNil(t, sampler)
 }

cmd/fleet/serve.go

@@ -85,6 +85,7 @@ import (
 	platform_http "github.com/fleetdm/fleet/v4/server/platform/http"
 	platform_logging "github.com/fleetdm/fleet/v4/server/platform/logging"
 	common_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql"
+	"github.com/fleetdm/fleet/v4/server/platform/tracing"
 	"github.com/fleetdm/fleet/v4/server/pubsub"
 	"github.com/fleetdm/fleet/v4/server/service"
 	"github.com/fleetdm/fleet/v4/server/service/async"
@@ -181,8 +182,16 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev
 	// Validate OTEL server options
 	config.Logging.Validate(initFatal)
 
-	// Init OTEL providers (traces, metrics, logs)
-	loggerProvider, tracerProvider, meterProvider := initOTELProviders(config, initFatal)
+	// Trace sampler tier registry. Bounded contexts register their own route to tier classifications below. The
+	// platform/tracing package stays free of cross context coupling. Infra paths (/healthz, /version, /metrics) are registered
+	// alongside their WrapHandler calls further down in this function.
+	traceRegistry := tracing.NewRegistry()
+	service.RegisterTracingTiers(traceRegistry)
+	activity_bootstrap.RegisterTracingTiers(traceRegistry)
+	// Future bounded contexts: each exposes its own RegisterTracingTiers.
+
+	// Init OTEL providers (traces, metrics, logs) and the route aware sampler.
+	loggerProvider, tracerProvider, meterProvider, traceSampler := initOTELProviders(config, traceRegistry, initFatal)
 
 	logger := initLogger(config, loggerProvider)
 
@@ -1037,6 +1046,13 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev
 		}
 	}()
 
+	// Trace sampler runtime control. The poller re-reads trace_sampler_settings every 60s and atomically swaps the sampler's
+	// ratios and force_full so support can flip a 100% debug window via PATCH /debug/trace_sampler without restarting any
+	// replicas. No-op when OTEL is disabled.
+	if traceSampler != nil {
+		go tracing.StartSettingsPoller(ctx, traceSampler, ds, logger)
+	}
+
 	if softwareInstallStore != nil {
 		if err := cronSchedules.StartCronSchedule(
 			func() (fleet.CronSchedule, error) {
@@ -1474,6 +1490,13 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev
 	), healthCheckers)
 
 	rootMux := http.NewServeMux()
+	// Infra paths (liveness, version, metrics) are platform owned and have zero diagnostic value as traces once their metric
+	// counterparts exist. Drop them unconditionally, even under ForceFull, since a 100% debug window should not flood SigNoz
+	// with probe spans. Register /metrics here even though its handler is mounted conditionally below. Registering a route
+	// whose handler isn't installed is a harmless lookup table entry and keeps the policy in one place.
+	traceRegistry.Register(http.MethodGet, "/healthz", tracing.TierNever)
+	traceRegistry.Register(http.MethodGet, "/version", tracing.TierNever)
+	traceRegistry.Register(http.MethodGet, "/metrics", tracing.TierNever)
 	rootMux.Handle("/healthz", service.PrometheusMetricsHandler("healthz", otelmw.WrapHandler(health.Handler(httpLogger, healthCheckers), "/healthz", config)))
 	rootMux.Handle("/version", service.PrometheusMetricsHandler("version", otelmw.WrapHandler(version.Handler(), "/version", config)))
 	rootMux.Handle("/assets/", service.PrometheusMetricsHandler("static_assets", otelmw.WrapHandlerDynamic(service.ServeStaticAssets("/assets/", serveCSP), config)))

server/activity/bootstrap/bootstrap.go

@@ -12,6 +12,7 @@ import (
 	platform_authz "github.com/fleetdm/fleet/v4/server/platform/authz"
 	eu "github.com/fleetdm/fleet/v4/server/platform/endpointer"
 	platform_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql"
+	"github.com/fleetdm/fleet/v4/server/platform/tracing"
 	"github.com/go-kit/kit/endpoint"
 )
 
@@ -31,3 +32,8 @@ func New(
 
 	return svc, routesFn
 }
+
+// RegisterTracingTiers classifies the activity context's routes for trace sampling.
+func RegisterTracingTiers(registry *tracing.Registry) {
+	service.RegisterTracingTiers(registry)
+}

server/activity/internal/service/handler.go

@@ -2,11 +2,13 @@ package service
 
 import (
 	"context"
+	"net/http"
 
 	"github.com/fleetdm/fleet/v4/server/activity/api"
 	api_http "github.com/fleetdm/fleet/v4/server/activity/api/http"
 	eu "github.com/fleetdm/fleet/v4/server/platform/endpointer"
 	platform_http "github.com/fleetdm/fleet/v4/server/platform/http"
+	"github.com/fleetdm/fleet/v4/server/platform/tracing"
 	"github.com/go-kit/kit/endpoint"
 	kithttp "github.com/go-kit/kit/transport/http"
 	"github.com/gorilla/mux"
@@ -27,6 +29,15 @@ func attachFleetAPIRoutes(r *mux.Router, svc api.Service, authMiddleware endpoin
 	ue.GET("/api/_version_/fleet/hosts/{id:[0-9]+}/activities", listHostPastActivitiesEndpoint, api_http.ListHostPastActivitiesRequest{})
 }
 
+// RegisterTracingTiers classifies this context's routes for trace sampling. Both activity list endpoints are admin reads, so
+// they belong in the standard tier (default 2%). Kept next to the route registrations above so the two stay in sync. The
+// sampler's version normalizer collapses the {fleetversion:...} segment, so the "_version_" placeholder matches the rendered
+// span name regardless of the configured API versions.
+func RegisterTracingTiers(registry *tracing.Registry) {
+	registry.Register(http.MethodGet, "/api/_version_/fleet/activities", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/hosts/{id}/activities", tracing.TierStandard)
+}
+
 func apiVersions() []string {
 	return []string{"v1", "latest"}
 }

server/datastore/mysql/migrations/tables/20260601200727_AddTraceSamplerSettings.go

@@ -0,0 +1,39 @@
+package tables
+
+import (
+	"database/sql"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20260601200727, Down_20260601200727)
+}
+
+func Up_20260601200727(tx *sql.Tx) error {
+	// Singleton settings row holding the runtime tunable trace sampling configuration. Operators flip these via PATCH
+	// /debug/trace_sampler. The /debug auth log and the PATCH access log already record who made the change.
+	_, err := tx.Exec(`
+		CREATE TABLE trace_sampler_settings (
+			id                TINYINT UNSIGNED NOT NULL PRIMARY KEY,
+			high_volume_ratio DOUBLE NOT NULL DEFAULT 0.001,
+			standard_ratio    DOUBLE NOT NULL DEFAULT 0.02,
+			force_full        TINYINT(1) NOT NULL DEFAULT 0,
+			updated_at        TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+			CONSTRAINT ck_trace_sampler_settings_singleton  CHECK (id = 1),
+			CONSTRAINT ck_trace_sampler_settings_high_range CHECK (high_volume_ratio BETWEEN 0 AND 1),
+			CONSTRAINT ck_trace_sampler_settings_std_range  CHECK (standard_ratio BETWEEN 0 AND 1)
+		) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
+	`)
+	if err != nil {
+		return err
+	}
+
+	// Insert the seed row with an explicit fixed updated_at instead of letting the column default to CURRENT_TIMESTAMP.
+	// schema.sql captures the row's data; if updated_at varied per migration run, every developer would get a different schema
+	// dump and `make test-schema` would never pass.
+	_, err = tx.Exec(`INSERT INTO trace_sampler_settings (id, updated_at) VALUES (1, '2026-06-01 00:00:00')`)
+	return err
+}
+
+func Down_20260601200727(tx *sql.Tx) error {
+	return nil
+}

server/datastore/mysql/trace_sampler.go

@@ -0,0 +1,56 @@
+package mysql
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+
+	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
+	"github.com/fleetdm/fleet/v4/server/platform/tracing"
+	"github.com/jmoiron/sqlx"
+)
+
+const traceSamplerSettingsID = 1
+
+func (ds *Datastore) GetTraceSamplerSettings(ctx context.Context) (*tracing.Settings, error) {
+	const stmt = `
+		SELECT high_volume_ratio, standard_ratio, force_full, updated_at
+		FROM trace_sampler_settings
+		WHERE id = ?
+	`
+	var settings tracing.Settings
+	if err := sqlx.GetContext(ctx, ds.reader(ctx), &settings, stmt, traceSamplerSettingsID); err != nil {
+		if errors.Is(err, sql.ErrNoRows) {
+			return nil, ctxerr.Wrap(ctx, notFound("TraceSamplerSettings"))
+		}
+		return nil, ctxerr.Wrap(ctx, err, "get trace_sampler_settings")
+	}
+	return &settings, nil
+}
+
+func (ds *Datastore) SetTraceSamplerSettings(ctx context.Context, settings *tracing.Settings) error {
+	const stmt = `
+		UPDATE trace_sampler_settings
+		SET high_volume_ratio = ?, standard_ratio = ?, force_full = ?
+		WHERE id = ?
+	`
+	res, err := ds.writer(ctx).ExecContext(ctx, stmt,
+		settings.HighVolumeRatio,
+		settings.StandardRatio,
+		settings.ForceFull,
+		traceSamplerSettingsID,
+	)
+	if err != nil {
+		return ctxerr.Wrap(ctx, err, "set trace_sampler_settings")
+	}
+	rows, err := res.RowsAffected()
+	if err != nil {
+		return ctxerr.Wrap(ctx, err, "set trace_sampler_settings: rows affected")
+	}
+	// The singleton row is seeded by the migration. A missing row means the invariant is broken.
+	if rows != 1 {
+		return ctxerr.Wrap(ctx, fmt.Errorf("set trace_sampler_settings: expected 1 row updated, got %d", rows))
+	}
+	return nil
+}

server/datastore/mysql/trace_sampler_test.go

@@ -0,0 +1,65 @@
+package mysql
+
+import (
+	"testing"
+
+	"github.com/fleetdm/fleet/v4/server/platform/tracing"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTraceSamplerSettings(t *testing.T) {
+	ds := CreateMySQLDS(t)
+	ctx := t.Context()
+
+	t.Run("seeded defaults are returned", func(t *testing.T) {
+		got, err := ds.GetTraceSamplerSettings(ctx)
+		require.NoError(t, err)
+		require.NotNil(t, got)
+		require.InDelta(t, 0.001, got.HighVolumeRatio, 1e-9)
+		require.InDelta(t, 0.02, got.StandardRatio, 1e-9)
+		require.False(t, got.ForceFull)
+	})
+
+	t.Run("round trip persists changes", func(t *testing.T) {
+		err := ds.SetTraceSamplerSettings(ctx, &tracing.Settings{
+			HighVolumeRatio: 0.005,
+			StandardRatio:   0.1,
+			ForceFull:       true,
+		})
+		require.NoError(t, err)
+
+		got, err := ds.GetTraceSamplerSettings(ctx)
+		require.NoError(t, err)
+		require.InDelta(t, 0.005, got.HighVolumeRatio, 1e-9)
+		require.InDelta(t, 0.1, got.StandardRatio, 1e-9)
+		require.True(t, got.ForceFull)
+	})
+
+	t.Run("out of range ratio rejected by CHECK constraint", func(t *testing.T) {
+		err := ds.SetTraceSamplerSettings(ctx, &tracing.Settings{
+			HighVolumeRatio: 1.5,
+			StandardRatio:   0.02,
+			ForceFull:       false,
+		})
+		require.Error(t, err)
+	})
+
+	t.Run("set fails when singleton row is missing", func(t *testing.T) {
+		// Locks in the RowsAffected != 1 guard in SetTraceSamplerSettings. If the seeded singleton row is missing (DB invariant
+		// broken), Set must surface a loud error rather than silently no-op.
+		_, err := ds.writer(ctx).ExecContext(ctx, `DELETE FROM trace_sampler_settings`)
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			_, err := ds.writer(ctx).ExecContext(ctx, `INSERT INTO trace_sampler_settings (id) VALUES (1)`)
+			require.NoError(t, err)
+		})
+
+		err = ds.SetTraceSamplerSettings(ctx, &tracing.Settings{
+			HighVolumeRatio: 0.01,
+			StandardRatio:   0.05,
+			ForceFull:       false,
+		})
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "expected 1 row updated")
+	})
+}

server/fleet/datastore.go

@@ -20,6 +20,7 @@ import (
 	"github.com/fleetdm/fleet/v4/server/mdm/nanomdm/storage"
 	platform_errors "github.com/fleetdm/fleet/v4/server/platform/errors"
 	platform_http "github.com/fleetdm/fleet/v4/server/platform/http"
+	"github.com/fleetdm/fleet/v4/server/platform/tracing"
 	"github.com/jmoiron/sqlx"
 )
 
@@ -3344,6 +3345,16 @@ type Datastore interface {
 
 	// HasWindowsUpdateConfigProfileConfigured checks if a profile for the team already exists in the update_settings table.
 	HasWindowsUpdateConfigProfileConfigured(ctx context.Context, teamID uint) (bool, error)
+
+	///////////////////////////////////////////////////////////////////////////////
+	// TraceSamplerStore
+
+	// GetTraceSamplerSettings returns the singleton trace_sampler_settings row.
+	GetTraceSamplerSettings(ctx context.Context) (*tracing.Settings, error)
+
+	// SetTraceSamplerSettings updates the singleton trace_sampler_settings row. The caller is responsible for validating that
+	// ratios are in [0, 1]. The DB CHECK constraints reject out of range writes as a backstop.
+	SetTraceSamplerSettings(ctx context.Context, settings *tracing.Settings) error
 }
 
 type AndroidDatastore interface {

server/mock/datastore_mock.go

@@ -18,6 +18,7 @@ import (
 	"github.com/fleetdm/fleet/v4/server/mdm/apple/mobileconfig"
 	"github.com/fleetdm/fleet/v4/server/mdm/nanodep/godep"
 	"github.com/fleetdm/fleet/v4/server/mdm/nanomdm/mdm"
+	"github.com/fleetdm/fleet/v4/server/platform/tracing"
 	"github.com/jmoiron/sqlx"
 )
 
@@ -2047,6 +2048,10 @@ type HasAppleUpdateConfigProfileConfiguredFunc func(ctx context.Context, teamID
 
 type HasWindowsUpdateConfigProfileConfiguredFunc func(ctx context.Context, teamID uint) (bool, error)
 
+type GetTraceSamplerSettingsFunc func(ctx context.Context) (*tracing.Settings, error)
+
+type SetTraceSamplerSettingsFunc func(ctx context.Context, settings *tracing.Settings) error
+
 type DataStore struct {
 	AppConfigFunc        AppConfigFunc
 	AppConfigFuncInvoked bool
@@ -5084,6 +5089,12 @@ type DataStore struct {
 	HasWindowsUpdateConfigProfileConfiguredFunc        HasWindowsUpdateConfigProfileConfiguredFunc
 	HasWindowsUpdateConfigProfileConfiguredFuncInvoked bool
 
+	GetTraceSamplerSettingsFunc        GetTraceSamplerSettingsFunc
+	GetTraceSamplerSettingsFuncInvoked bool
+
+	SetTraceSamplerSettingsFunc        SetTraceSamplerSettingsFunc
+	SetTraceSamplerSettingsFuncInvoked bool
+
 	mu sync.Mutex
 }
 
@@ -12170,3 +12181,17 @@ func (s *DataStore) HasWindowsUpdateConfigProfileConfigured(ctx context.Context,
 	s.mu.Unlock()
 	return s.HasWindowsUpdateConfigProfileConfiguredFunc(ctx, teamID)
 }
+
+func (s *DataStore) GetTraceSamplerSettings(ctx context.Context) (*tracing.Settings, error) {
+	s.mu.Lock()
+	s.GetTraceSamplerSettingsFuncInvoked = true
+	s.mu.Unlock()
+	return s.GetTraceSamplerSettingsFunc(ctx)
+}
+
+func (s *DataStore) SetTraceSamplerSettings(ctx context.Context, settings *tracing.Settings) error {
+	s.mu.Lock()
+	s.SetTraceSamplerSettingsFuncInvoked = true
+	s.mu.Unlock()
+	return s.SetTraceSamplerSettingsFunc(ctx, settings)
+}

server/platform/tracing/doc.go

@@ -0,0 +1,13 @@
+// Package tracing implements a route aware head sampler for Fleet's OTEL trace export. The sampler classifies each span via a
+// Registry. Each bounded context populates the Registry at startup with its own routes. The sampler then applies per tier ratio
+// sampling. The goal is to prevent noisy hot agent paths from drowning out the rare load bearing paths (enroll, MDM command
+// flows, cron jobs).
+//
+// Runtime control lives in the trace_sampler_settings MySQL row. Each Fleet replica runs StartSettingsPoller which re-reads the
+// row every 60 seconds and atomically swaps the sampler's state. No restart is required to flip force_full during an incident
+// debug window.
+//
+// Architecture: platform/tracing owns the mechanism (sampler, tier enum, registry). Each bounded context owns the policy: which
+// of its routes belong in which tier. Each context registers them at startup. This keeps the platform package free of cross
+// context coupling.
+package tracing

server/platform/tracing/registry.go

@@ -0,0 +1,92 @@
+package tracing
+
+import (
+	"regexp"
+	"sync"
+)
+
+// Tier classifies a span for sampling. Higher value spans (TierAlways) always get sampled. High volume noise (TierHighVolume)
+// is downsampled aggressively. Liveness probes (TierNever) are dropped unconditionally.
+type Tier int
+
+const (
+	// TierAlways is the catch all. Spans not registered with the Registry fall here, and are sampled at 100%. Cron jobs, enroll,
+	// MDM checkin, SCEP, GitOps batch (all the load bearing flows) should never need an explicit registration. Their absence from
+	// the noisy lists is what keeps them safe by default.
+	TierAlways Tier = iota
+
+	// TierHighVolume is for routes that dominate request volume without being individually interesting (osquery distributed
+	// read/write, orbit ping/config, device desktop/ping). Sampled at the configured high volume ratio (default 0.1%).
+	TierHighVolume
+
+	// TierStandard is for routes with moderate volume and moderate diagnostic value (admin reads, dashboard endpoints, asset
+	// paths). Sampled at the configured standard ratio (default 2%).
+	TierStandard
+
+	// TierNever drops the span unconditionally, even under ForceFull. Reserved for high volume zero diagnostic value paths like
+	// liveness probes, the version endpoint, and the Prometheus scrape path.
+	TierNever
+)
+
+// Registry maps normalized route names to tiers. Bounded contexts register their own routes at startup via Register. The
+// sampler reads via Lookup on every span. Routes not present in the registry fall to TierAlways.
+//
+// The Registry is the policy seam. platform/tracing provides the mechanism (sampler and tier enum). Each bounded context
+// provides the policy: which of its routes belong in which tier.
+type Registry struct {
+	mu     sync.RWMutex
+	routes map[string]Tier
+}
+
+// NewRegistry returns an empty Registry. Routes are added via Register.
+func NewRegistry() *Registry {
+	return &Registry{routes: make(map[string]Tier)}
+}
+
+// Register classifies a method and path. Paths are normalized on write the same way they are on Lookup. The gorilla/mux
+// version segment is collapsed to "_version_". Regex constrained params (e.g. "{id:[0-9]+}") are stripped to "{id}". Callers
+// can therefore register in either the readable form ("/api/_version_/fleet/hosts/{id}") or the raw template form, and lookups
+// will still match. Both alternate path forms (e.g. "/api/v1/osquery/..." and "/api/osquery/...") still need separate
+// registrations.
+//
+// Re-registering the same normalized method+path overwrites the prior tier.
+func (r *Registry) Register(method, path string, tier Tier) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.routes[normalizeSpanName(spanKey(method, path))] = tier
+}
+
+// Lookup returns the tier for a span name and a bool indicating whether the route was found in the registry. Span names are
+// normalized before lookup so the gorilla version regex is stripped to "_version_". Cron and other non HTTP span names (no
+// leading "METHOD ") simply won't be in the registry and return (TierAlways, false).
+func (r *Registry) Lookup(spanName string) (Tier, bool) {
+	normalized := normalizeSpanName(spanName)
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	t, ok := r.routes[normalized]
+	return t, ok
+}
+
+// versionTemplatePattern matches the gorilla/mux version segment Fleet inserts via /_version_/, replacing it with
+// /{fleetversion:(?:v1|2022-04|latest)}/ at registration time. We normalize it back to /_version_/ so the registry stays
+// decoupled from the configured version list.
+var versionTemplatePattern = regexp.MustCompile(`\{fleetversion:[^}]*\}`)
+
+// muxParamRegexPattern strips regex constraints from gorilla/mux path params. For example {id:[0-9]+} becomes {id}, and
+// {fleet_id:[0-9]+} becomes {fleet_id}. Fleet uses this style extensively, and the route tier policy in
+// server/service/tracing_tiers.go registers the simpler {id} form. Without this normalization, spans whose mux template
+// includes the regex would miss the registry and silently fall to TierAlways at 100% sampling.
+var muxParamRegexPattern = regexp.MustCompile(`\{([a-zA-Z0-9_]+):[^}]+\}`)
+
+func normalizeSpanName(name string) string {
+	// Apply versionTemplatePattern first because it strips both the param name and the surrounding braces (replacing with
+	// `_version_`). muxParamRegexPattern preserves the braces. Running it first would turn {fleetversion:...} into
+	// {fleetversion}, which would then no longer match versionTemplatePattern.
+	name = versionTemplatePattern.ReplaceAllString(name, "_version_")
+	name = muxParamRegexPattern.ReplaceAllString(name, "{$1}")
+	return name
+}
+
+func spanKey(method, path string) string {
+	return method + " " + path
+}

server/platform/tracing/registry_test.go

@@ -0,0 +1,176 @@
+package tracing
+
+import (
+	"net/http"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRegistry_LookupNormalizes(t *testing.T) {
+	// One fixture covers all input shapes that should resolve to the same registered route: fleetversion templates (multi
+	// version and single version), mux regex constrained params, and already normalized inputs.
+	r := NewRegistry()
+	r.Register(http.MethodGet, "/api/_version_/fleet/hosts", TierStandard)
+	r.Register(http.MethodGet, "/api/_version_/fleet/hosts/{id}", TierStandard)
+	r.Register(http.MethodPatch, "/api/_version_/fleet/fleets/{fleet_id}/secrets", TierAlways)
+
+	cases := []struct {
+		name   string
+		input  string
+		want   Tier
+		wantOK bool
+	}{
+		{
+			name:   "regex version template",
+			input:  "GET /api/{fleetversion:(?:v1|2022-04|latest)}/fleet/hosts",
+			want:   TierStandard,
+			wantOK: true,
+		},
+		{
+			name:   "single version regex template",
+			input:  "GET /api/{fleetversion:(?:latest)}/fleet/hosts",
+			want:   TierStandard,
+			wantOK: true,
+		},
+		{
+			name:   "already normalized form",
+			input:  "GET /api/_version_/fleet/hosts",
+			want:   TierStandard,
+			wantOK: true,
+		},
+		{
+			name:   "mux regex param {id:[0-9]+}",
+			input:  "GET /api/{fleetversion:(?:v1|2022-04|latest)}/fleet/hosts/{id:[0-9]+}",
+			want:   TierStandard,
+			wantOK: true,
+		},
+		{
+			name:   "mux regex param {fleet_id:[0-9]+}",
+			input:  "PATCH /api/{fleetversion:(?:v1|2022-04|latest)}/fleet/fleets/{fleet_id:[0-9]+}/secrets",
+			want:   TierAlways,
+			wantOK: true,
+		},
+		{
+			name:   "unregistered route",
+			input:  "POST /not/in/registry",
+			want:   TierAlways, // zero value. Sampler interprets the !ok as the catch all.
+			wantOK: false,
+		},
+		{
+			name:   "cron span name is not in registry",
+			input:  "vuln.update_host_counts",
+			want:   TierAlways,
+			wantOK: false,
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got, ok := r.Lookup(c.input)
+			require.Equal(t, c.wantOK, ok)
+			require.Equal(t, c.want, got)
+		})
+	}
+}
+
+func TestRegistry_RegisterNormalizesSymmetrically(t *testing.T) {
+	// Inverse of TestRegistry_LookupNormalizes: registering with the regex form must also resolve via lookup of the bare form.
+	// This is the invariant the Register side normalization delivers.
+	r := NewRegistry()
+	r.Register(http.MethodGet, "/api/_version_/fleet/hosts/{id:[0-9]+}", TierStandard)
+
+	got, ok := r.Lookup("GET /api/{fleetversion:(?:latest)}/fleet/hosts/{id}")
+	require.True(t, ok, "regex form registration must be findable via simple form lookup")
+	require.Equal(t, TierStandard, got)
+}
+
+func TestRegistry_RegisterOverwrites(t *testing.T) {
+	t.Run("same form", func(t *testing.T) {
+		r := NewRegistry()
+		r.Register(http.MethodPost, "/foo", TierStandard)
+		r.Register(http.MethodPost, "/foo", TierHighVolume)
+
+		got, ok := r.Lookup("POST /foo")
+		require.True(t, ok)
+		require.Equal(t, TierHighVolume, got)
+	})
+
+	t.Run("different forms collide on the same normalized key", func(t *testing.T) {
+		// {id} and {id:[0-9]+} are the same logical route after normalization. The second Register must overwrite the first
+		// regardless of which surface form was used.
+		r := NewRegistry()
+		r.Register(http.MethodGet, "/api/_version_/fleet/hosts/{id}", TierStandard)
+		r.Register(http.MethodGet, "/api/_version_/fleet/hosts/{id:[0-9]+}", TierHighVolume)
+
+		got, _ := r.Lookup("GET /api/_version_/fleet/hosts/{id}")
+		require.Equal(t, TierHighVolume, got)
+	})
+}
+
+func TestRegistry_ConcurrentReadersAndWriters(t *testing.T) {
+	// Exercise the RWMutex under -race. Late arriving registrations must not corrupt concurrent lookups. This is what makes it
+	// safe for bounded contexts to register at startup while the tracer provider is already serving spans.
+	r := NewRegistry()
+	const writerCount = 4
+	const readerCount = 8
+	const iterations = 5000
+
+	var wg sync.WaitGroup
+	for w := range writerCount {
+		wg.Go(func() {
+			for i := range iterations {
+				path := "/path/writer/" + strconv.Itoa(w) + "/" + strconv.Itoa(i)
+				r.Register(http.MethodGet, path, TierStandard)
+			}
+		})
+	}
+	for range readerCount {
+		wg.Go(func() {
+			for range iterations {
+				_, _ = r.Lookup("GET /path/writer/0/0")
+			}
+		})
+	}
+	wg.Wait()
+}
+
+func TestNormalizeSpanName(t *testing.T) {
+	// Unit test for the helper. Lookup integration is covered separately. These cases pin the helper's behavior independent of
+	// the map lookup.
+	cases := []struct {
+		in, want string
+	}{
+		{"GET /healthz", "GET /healthz"},
+		{
+			"GET /api/{fleetversion:(?:v1|2022-04|latest)}/fleet/hosts",
+			"GET /api/_version_/fleet/hosts",
+		},
+		{
+			"GET /api/_version_/fleet/queries",
+			"GET /api/_version_/fleet/queries",
+		},
+		{"vuln.update_host_counts", "vuln.update_host_counts"},
+		// Mux regex constraints on path params are stripped to the bare {name} form so the registry can stay decoupled from the
+		// constraint syntax.
+		{
+			"GET /api/_version_/fleet/hosts/{id:[0-9]+}",
+			"GET /api/_version_/fleet/hosts/{id}",
+		},
+		{
+			"PATCH /api/{fleetversion:(?:v1|2022-04|latest)}/fleet/fleets/{fleet_id:[0-9]+}/secrets",
+			"PATCH /api/_version_/fleet/fleets/{fleet_id}/secrets",
+		},
+		// Params without a regex constraint pass through unchanged.
+		{
+			"GET /api/_version_/fleet/device/{token}/desktop",
+			"GET /api/_version_/fleet/device/{token}/desktop",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.in, func(t *testing.T) {
+			require.Equal(t, c.want, normalizeSpanName(c.in))
+		})
+	}
+}

server/platform/tracing/sampler.go

@@ -0,0 +1,105 @@
+package tracing
+
+import (
+	"fmt"
+	"sync/atomic"
+
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+// Default sampling ratios. These match the seeded trace_sampler_settings row so a freshly started server uses the same ratios
+// as one that has polled the DB.
+const (
+	DefaultHighVolumeRatio = 0.001
+	DefaultStandardRatio   = 0.02
+)
+
+// RouteTierSampler implements sdktrace.Sampler. The configured ratios live in an atomic.Pointer so SettingsPoller can swap them
+// under a hot reader without locking. Tier classification is delegated to the Registry passed at construction time.
+type RouteTierSampler struct {
+	state    atomic.Pointer[samplerState]
+	registry *Registry
+}
+
+type samplerState struct {
+	highVolume sdktrace.Sampler
+	standard   sdktrace.Sampler
+	always     sdktrace.Sampler
+	never      sdktrace.Sampler
+	forceFull  bool
+}
+
+// NewRouteTierSampler returns a sampler initialized with the default ratios and force_full=false. The poller (if running)
+// overwrites these on the first tick once it reads the DB row. The registry is consulted on every ShouldSample call. Routes
+// added to the registry after construction are picked up immediately.
+func NewRouteTierSampler(registry *Registry) *RouteTierSampler {
+	s := &RouteTierSampler{registry: registry}
+	s.state.Store(buildState(DefaultHighVolumeRatio, DefaultStandardRatio, false))
+	return s
+}
+
+// Apply replaces the sampler's state atomically. Out of range ratios are clamped to [0, 1] as a defensive backstop. The DB
+// CHECK constraints and the PATCH handler validation reject these earlier in the pipeline.
+func (s *RouteTierSampler) Apply(highVolume, standard float64, forceFull bool) {
+	s.state.Store(buildState(clamp01(highVolume), clamp01(standard), forceFull))
+}
+
+func clamp01(v float64) float64 {
+	switch {
+	case v < 0:
+		return 0
+	case v > 1:
+		return 1
+	}
+	return v
+}
+
+func buildState(highVolume, standard float64, forceFull bool) *samplerState {
+	return &samplerState{
+		highVolume: sdktrace.TraceIDRatioBased(highVolume),
+		standard:   sdktrace.TraceIDRatioBased(standard),
+		always:     sdktrace.AlwaysSample(),
+		never:      sdktrace.NeverSample(),
+		forceFull:  forceFull,
+	}
+}
+
+// ShouldSample implements sdktrace.Sampler. TierNever wins over ForceFull. Liveness probes should never trace, even during a
+// 100% debug window. Unregistered spans (cron, MDM checkin, enroll, etc.) fall to TierAlways.
+func (s *RouteTierSampler) ShouldSample(p sdktrace.SamplingParameters) sdktrace.SamplingResult {
+	st := s.state.Load()
+	tier, _ := s.registry.Lookup(p.Name)
+	if tier == TierNever {
+		return st.never.ShouldSample(p)
+	}
+	if st.forceFull {
+		return st.always.ShouldSample(p)
+	}
+	switch tier {
+	case TierHighVolume:
+		return st.highVolume.ShouldSample(p)
+	case TierStandard:
+		return st.standard.ShouldSample(p)
+	case TierAlways:
+		return st.always.ShouldSample(p)
+	}
+	return st.always.ShouldSample(p)
+}
+
+// Description implements sdktrace.Sampler. OTel uses it for diagnostic logging. The value should describe the sampler's
+// behavior unambiguously.
+func (s *RouteTierSampler) Description() string {
+	st := s.state.Load()
+	return fmt.Sprintf("RouteTierSampler{highVolume=%g,standard=%g,forceFull=%t}",
+		samplerRatio(st.highVolume), samplerRatio(st.standard), st.forceFull)
+}
+
+// samplerRatio extracts the configured ratio from a TraceIDRatioBased sampler for description purposes only. The SDK does not
+// expose the ratio directly, so we parse its description ("TraceIDRatioBased{0.001}").
+func samplerRatio(s sdktrace.Sampler) float64 {
+	var r float64
+	if _, err := fmt.Sscanf(s.Description(), "TraceIDRatioBased{%f}", &r); err != nil {
+		return -1
+	}
+	return r
+}

server/platform/tracing/sampler_test.go

@@ -0,0 +1,200 @@
+package tracing
+
+import (
+	"encoding/binary"
+	"math/rand/v2"
+	"net/http"
+	"sync"
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// testRegistry returns a Registry pre-populated with the routes the sampler tests exercise. It lives next to its only consumer
+// so registry_test.go can stay focused on Registry semantics.
+func testRegistry() *Registry {
+	r := NewRegistry()
+	r.Register(http.MethodGet, "/healthz", TierNever)
+	r.Register(http.MethodGet, "/version", TierNever)
+	r.Register(http.MethodGet, "/metrics", TierNever)
+	r.Register(http.MethodPost, "/api/osquery/distributed/read", TierHighVolume)
+	r.Register(http.MethodPost, "/api/v1/osquery/distributed/read", TierHighVolume)
+	r.Register(http.MethodPost, "/api/osquery/distributed/write", TierHighVolume)
+	r.Register(http.MethodPost, "/api/fleet/orbit/config", TierHighVolume)
+	r.Register(http.MethodHead, "/api/fleet/orbit/ping", TierHighVolume)
+	r.Register(http.MethodHead, "/api/_version_/fleet/device/{token}/ping", TierHighVolume)
+	r.Register(http.MethodGet, "/api/_version_/fleet/device/{token}/desktop", TierHighVolume)
+	r.Register(http.MethodGet, "/api/_version_/fleet/hosts", TierStandard)
+	r.Register(http.MethodGet, "/api/_version_/fleet/hosts/{id}", TierStandard)
+	r.Register(http.MethodGet, "/api/_version_/fleet/queries", TierStandard)
+	return r
+}
+
+// sample is a tiny helper that asks the sampler whether a span with the given name should be recorded. A pseudo random trace
+// ID is used so the TraceIDRatioBased sampler's decision varies per call.
+//
+//nolint:gosec // test trace IDs, not security sensitive
+func sample(t *testing.T, s *RouteTierSampler, name string) bool {
+	t.Helper()
+	var tid trace.TraceID
+	binary.LittleEndian.PutUint64(tid[0:8], rand.Uint64())
+	binary.LittleEndian.PutUint64(tid[8:16], rand.Uint64())
+	res := s.ShouldSample(sdktrace.SamplingParameters{
+		TraceID: tid,
+		Name:    name,
+		Kind:    trace.SpanKindServer,
+	})
+	return res.Decision == sdktrace.RecordAndSample
+}
+
+// sampleRate runs N trials and returns the observed sample rate.
+func sampleRate(t *testing.T, s *RouteTierSampler, name string, n int) float64 {
+	t.Helper()
+	hits := 0
+	for range n {
+		if sample(t, s, name) {
+			hits++
+		}
+	}
+	return float64(hits) / float64(n)
+}
+
+// TestRouteTierSampler_NeverTierDropsUnconditionally locks in the invariant that TierNever paths are never sampled, both at
+// default config and under the most aggressive override (force_full=true with ratios maxed). The force_full subtest is the
+// stronger guarantee. The default config case is kept to make the absence of any default-time leak explicit.
+func TestRouteTierSampler_NeverTierDropsUnconditionally(t *testing.T) {
+	paths := []string{"GET /healthz", "GET /version", "GET /metrics"}
+
+	cases := []struct {
+		name  string
+		apply func(*RouteTierSampler)
+	}{
+		{name: "default config", apply: func(*RouteTierSampler) {}},
+		{name: "force_full with max ratios", apply: func(s *RouteTierSampler) { s.Apply(1.0, 1.0, true) }},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			s := NewRouteTierSampler(testRegistry())
+			c.apply(s)
+			for _, p := range paths {
+				for range 1000 {
+					require.False(t, sample(t, s, p), "tierNever must drop %s", p)
+				}
+			}
+		})
+	}
+}
+
+// TestRouteTierSampler_AlwaysTierKeepsUnconditionally locks in the invariant that unclassified spans (cron, novel routes)
+// always sample, both at default config and when ratios are forced to zero. The ratios=0 subtest is the stronger guarantee.
+func TestRouteTierSampler_AlwaysTierKeepsUnconditionally(t *testing.T) {
+	names := []string{
+		"vuln.update_host_counts",                      // cron
+		"POST /api/_version_/fleet/mdm/profiles/batch", // GitOps batch
+		"POST /api/fleet/orbit/enroll",                 // enroll
+		"some-future-endpoint-not-in-any-list",         // unknown
+	}
+
+	cases := []struct {
+		name  string
+		apply func(*RouteTierSampler)
+	}{
+		{name: "default config", apply: func(*RouteTierSampler) {}},
+		{name: "ratios forced to zero", apply: func(s *RouteTierSampler) { s.Apply(0.0, 0.0, false) }},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			s := NewRouteTierSampler(testRegistry())
+			c.apply(s)
+			for _, n := range names {
+				for range 1000 {
+					require.True(t, sample(t, s, n), "tierAlways must keep %s", n)
+				}
+			}
+		})
+	}
+}
+
+func TestRouteTierSampler_RatioSampling(t *testing.T) {
+	const n = 100_000
+	const tolerance = 0.005 // 0.5pp absolute, generous for 100k trials
+
+	s := NewRouteTierSampler(testRegistry())
+	s.Apply(0.1, 0.5, false) // visible ratios for stable comparisons
+
+	highRate := sampleRate(t, s, "POST /api/osquery/distributed/read", n)
+	require.InDelta(t, 0.1, highRate, tolerance, "high volume tier should track its configured ratio")
+
+	stdRate := sampleRate(t, s, "GET /api/_version_/fleet/hosts", n)
+	require.InDelta(t, 0.5, stdRate, tolerance, "standard tier should track its configured ratio")
+}
+
+func TestRouteTierSampler_ForceFull(t *testing.T) {
+	s := NewRouteTierSampler(testRegistry())
+	s.Apply(0.0, 0.0, true) // ratios zero, force_full should override
+
+	for _, name := range []string{
+		"POST /api/osquery/distributed/read", // would be 0% via high volume
+		"GET /api/_version_/fleet/hosts",     // would be 0% via standard
+		"POST /api/fleet/orbit/enroll",       // already always
+	} {
+		t.Run(name, func(t *testing.T) {
+			for range 1000 {
+				require.True(t, sample(t, s, name),
+					"force_full must override ratio based tiers")
+			}
+		})
+	}
+}
+
+func TestRouteTierSampler_ApplyRaceFree(t *testing.T) {
+	s := NewRouteTierSampler(testRegistry())
+
+	var (
+		stop    atomic.Bool
+		readers sync.WaitGroup
+		writers sync.WaitGroup
+	)
+
+	// One writer flips ratios continuously.
+	writers.Go(func() {
+		for !stop.Load() {
+			//nolint:gosec // test fuzz inputs, not security sensitive
+			s.Apply(rand.Float64(), rand.Float64(), rand.IntN(2) == 0)
+		}
+	})
+
+	// Several readers hammer ShouldSample.
+	const readerCount = 8
+	for range readerCount {
+		readers.Go(func() {
+			for !stop.Load() {
+				_ = sample(t, s, "POST /api/osquery/distributed/read")
+				_ = sample(t, s, "GET /healthz")
+				_ = sample(t, s, "vuln.update_host_counts")
+			}
+		})
+	}
+
+	// Run for a tight window. The race detector will fail if there's a torn read.
+	for range 50_000 {
+		_ = sample(t, s, "POST /api/_version_/fleet/spec/teams")
+	}
+	stop.Store(true)
+	readers.Wait()
+	writers.Wait()
+}
+
+func TestRouteTierSampler_ClampOutOfRange(t *testing.T) {
+	// Apply should clamp defensively even if a caller passes out of range ratios. The DB CHECK rejects these in practice.
+	s := NewRouteTierSampler(testRegistry())
+	s.Apply(-1.0, 5.0, false)
+	st := s.state.Load()
+	require.Equal(t, "TraceIDRatioBased{0}", st.highVolume.Description())
+	require.Equal(t, "TraceIDRatioBased{1}", st.standard.Description())
+}

server/platform/tracing/settings.go

@@ -0,0 +1,13 @@
+package tracing
+
+import "time"
+
+// Settings is the runtime tunable head sampling configuration. The settings live in the trace_sampler_settings singleton row
+// and are polled by each Fleet replica so support can adjust sampling without a restart.
+type Settings struct {
+	HighVolumeRatio float64 `json:"high_volume_ratio" db:"high_volume_ratio"`
+	StandardRatio   float64 `json:"standard_ratio" db:"standard_ratio"`
+	ForceFull       bool    `json:"force_full" db:"force_full"`
+	// UpdatedAt uses omitzero so the PATCH handler can zero it before echoing the response.
+	UpdatedAt time.Time `json:"updated_at,omitzero" db:"updated_at"`
+}

server/platform/tracing/settings_poller.go

@@ -0,0 +1,72 @@
+package tracing
+
+import (
+	"context"
+	"log/slog"
+	"time"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// settingsPollInterval is how often each Fleet replica re-reads trace_sampler_settings. 60s matches industry defaults for
+// feature flag style runtime config.
+const settingsPollInterval = 60 * time.Second
+
+// pollTracer instruments the poll loop. When OTEL is not configured the global provider returns a no-op tracer, so this is
+// free when tracing is disabled.
+var pollTracer = otel.Tracer("github.com/fleetdm/fleet/v4/server/platform/tracing")
+
+// settingsReader is the minimal datastore surface the poller needs. The full fleet.Datastore is large. Depending only on this
+// interface keeps the poller's tests cheap and the package free of cross context coupling.
+type settingsReader interface {
+	GetTraceSamplerSettings(ctx context.Context) (*Settings, error)
+}
+
+// StartSettingsPoller runs the polling loop until ctx is cancelled. On each tick it reads the trace_sampler_settings row,
+// compares the values to the last applied state, and calls sampler. Apply only when something changed.
+//
+// On startup it does one immediate read so the sampler picks up the row's current values without waiting a full interval. If
+// the first read fails, the sampler keeps its compile time defaults and a warning is logged. The next tick will try again.
+func StartSettingsPoller(ctx context.Context, sampler *RouteTierSampler, ds settingsReader, logger *slog.Logger) {
+	pollAndApply := func(last *Settings) *Settings {
+		spanCtx, span := pollTracer.Start(ctx, "tracing.poll_settings",
+			trace.WithNewRoot(),
+			trace.WithSpanKind(trace.SpanKindInternal),
+		)
+		defer span.End()
+
+		got, err := ds.GetTraceSamplerSettings(spanCtx)
+		if err != nil {
+			logger.ErrorContext(spanCtx, "trace sampler settings poll failed", "err", err)
+			return last
+		}
+		if last != nil &&
+			got.HighVolumeRatio == last.HighVolumeRatio &&
+			got.StandardRatio == last.StandardRatio &&
+			got.ForceFull == last.ForceFull {
+			return last
+		}
+		sampler.Apply(got.HighVolumeRatio, got.StandardRatio, got.ForceFull)
+		logger.InfoContext(spanCtx, "trace sampler settings applied",
+			"high_volume_ratio", got.HighVolumeRatio,
+			"standard_ratio", got.StandardRatio,
+			"force_full", got.ForceFull,
+		)
+		return got
+	}
+
+	var last *Settings
+	last = pollAndApply(last)
+
+	ticker := time.NewTicker(settingsPollInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			last = pollAndApply(last)
+		}
+	}
+}

server/platform/tracing/settings_poller_test.go

@@ -0,0 +1,139 @@
+package tracing
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"sync/atomic"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+type stubReader struct {
+	mu       atomic.Pointer[Settings]
+	err      atomic.Pointer[error]
+	getCalls atomic.Int32
+}
+
+func (s *stubReader) set(settings Settings) {
+	s.mu.Store(&settings)
+}
+
+func (s *stubReader) setErr(err error) {
+	s.err.Store(&err)
+}
+
+func (s *stubReader) GetTraceSamplerSettings(_ context.Context) (*Settings, error) {
+	s.getCalls.Add(1)
+	if e := s.err.Load(); e != nil && *e != nil {
+		return nil, *e
+	}
+	if cur := s.mu.Load(); cur != nil {
+		out := *cur
+		return &out, nil
+	}
+	return &Settings{
+		HighVolumeRatio: DefaultHighVolumeRatio,
+		StandardRatio:   DefaultStandardRatio,
+	}, nil
+}
+
+func discardLogger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(io.Discard, nil))
+}
+
+func TestStartSettingsPoller_AppliesInitialReadImmediately(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		r := &stubReader{}
+		r.set(Settings{
+			HighVolumeRatio: 0.4,
+			StandardRatio:   0.8,
+			ForceFull:       true,
+		})
+
+		sampler := NewRouteTierSampler(NewRegistry())
+		ctx, cancel := context.WithCancel(t.Context())
+		defer cancel()
+
+		go StartSettingsPoller(ctx, sampler, r, discardLogger())
+
+		// Wait blocks until every other goroutine in the bubble is durably blocked. The poller does its synchronous initial
+		// read, applies, then blocks on the ticker. So once Wait returns, the apply has happened.
+		synctest.Wait()
+
+		require.Equal(t, int32(1), r.getCalls.Load(), "exactly one poll should have happened by now")
+		st := sampler.state.Load()
+		require.True(t, st.forceFull, "initial read must apply force_full")
+	})
+}
+
+func TestStartSettingsPoller_HandlesErrorGracefully(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		r := &stubReader{}
+		r.setErr(errors.New("db unavailable"))
+
+		sampler := NewRouteTierSampler(NewRegistry())
+		// Capture current state. It should remain unchanged after a failed poll.
+		beforeForceFull := sampler.state.Load().forceFull
+
+		ctx, cancel := context.WithCancel(t.Context())
+		defer cancel()
+
+		go StartSettingsPoller(ctx, sampler, r, discardLogger())
+
+		synctest.Wait()
+
+		require.Equal(t, int32(1), r.getCalls.Load(), "the failed poll still counts as one read")
+		require.Equal(t, beforeForceFull, sampler.state.Load().forceFull,
+			"sampler state must be unchanged when the read fails")
+	})
+}
+
+func TestStartSettingsPoller_AppliesChangeOnTick(t *testing.T) {
+	// Locks in the actual 60s ticker behavior. The old tests could only assert the initial synchronous read because waiting a
+	// real minute per test was untenable. With synctest, advancing time is free.
+	synctest.Test(t, func(t *testing.T) {
+		r := &stubReader{}
+		r.set(Settings{
+			HighVolumeRatio: 0.4,
+			StandardRatio:   0.8,
+			ForceFull:       true,
+		})
+
+		sampler := NewRouteTierSampler(NewRegistry())
+		ctx, cancel := context.WithCancel(t.Context())
+		defer cancel()
+
+		go StartSettingsPoller(ctx, sampler, r, discardLogger())
+
+		// Initial synchronous poll completes.
+		synctest.Wait()
+		require.Equal(t, int32(1), r.getCalls.Load())
+		require.True(t, sampler.state.Load().forceFull)
+
+		// Flip the stub to a new value. Advance past the next ticker fire; the synthetic clock advances while everything is
+		// blocked, the ticker fires, the poller re-polls and applies the new state. Wait then ensures the poller has
+		// re-blocked before we assert.
+		r.set(Settings{
+			HighVolumeRatio: 0.001,
+			StandardRatio:   0.02,
+			ForceFull:       false,
+		})
+		time.Sleep(settingsPollInterval + time.Nanosecond)
+		synctest.Wait()
+
+		require.Equal(t, int32(2), r.getCalls.Load(), "second poll must have fired after one ticker interval")
+		require.False(t, sampler.state.Load().forceFull, "ticker poll must apply the new state")
+
+		// One more tick with no underlying change should still call Get but should be a no-op for Apply. We verify by the
+		// invariant that the state is unchanged from the previous assertion.
+		time.Sleep(settingsPollInterval + time.Nanosecond)
+		synctest.Wait()
+		require.Equal(t, int32(3), r.getCalls.Load())
+		require.False(t, sampler.state.Load().forceFull)
+	})
+}

server/service/debug_handler.go

@@ -11,6 +11,7 @@ import (
 	"github.com/fleetdm/fleet/v4/server/config"
 	"github.com/fleetdm/fleet/v4/server/contexts/logging"
 	"github.com/fleetdm/fleet/v4/server/contexts/token"
+	"github.com/fleetdm/fleet/v4/server/contexts/viewer"
 	"github.com/fleetdm/fleet/v4/server/errorstore"
 	"github.com/fleetdm/fleet/v4/server/fleet"
 	"github.com/fleetdm/fleet/v4/server/service/middleware/auth"
@@ -43,7 +44,9 @@ func (m *debugAuthenticationMiddleware) Middleware(next http.Handler) http.Handl
 			return
 		}
 
-		next.ServeHTTP(w, r)
+		// Attach the authenticated viewer to the request context so downstream debug handlers can record who triggered an
+		// action (e.g. updating trace sampler settings).
+		next.ServeHTTP(w, r.WithContext(viewer.NewContext(r.Context(), *v)))
 	})
 }
 
@@ -92,6 +95,10 @@ func MakeDebugHandler(svc fleet.Service, config config.FleetConfig, logger *slog
 	r.HandleFunc("/debug/db/locks", jsonHandler(logger, func(ctx context.Context) (interface{}, error) { return ds.DBLocks(ctx) }))
 	r.HandleFunc("/debug/db/innodb-status", jsonHandler(logger, func(ctx context.Context) (interface{}, error) { return ds.InnoDBStatus(ctx) }))
 	r.HandleFunc("/debug/db/process-list", jsonHandler(logger, func(ctx context.Context) (interface{}, error) { return ds.ProcessList(ctx) }))
+	r.HandleFunc("/debug/trace_sampler", jsonHandler(logger, func(ctx context.Context) (any, error) {
+		return ds.GetTraceSamplerSettings(ctx)
+	})).Methods(http.MethodGet)
+	r.HandleFunc("/debug/trace_sampler", patchTraceSamplerHandler(logger, ds)).Methods(http.MethodPatch)
 
 	mw := &debugAuthenticationMiddleware{
 		service: svc,

server/service/debug_trace_sampler.go

@@ -0,0 +1,113 @@
+package service
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"time"
+
+	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
+	"github.com/fleetdm/fleet/v4/server/contexts/viewer"
+	"github.com/fleetdm/fleet/v4/server/fleet"
+)
+
+// traceSamplerPatchRequest is the PATCH payload. Fields are pointers so we can distinguish "unset" from a zero value. PATCH
+// semantics mean only the provided fields are applied.
+type traceSamplerPatchRequest struct {
+	HighVolumeRatio *float64 `json:"high_volume_ratio,omitempty"`
+	StandardRatio   *float64 `json:"standard_ratio,omitempty"`
+	ForceFull       *bool    `json:"force_full,omitempty"`
+}
+
+// patchTraceSamplerHandler returns the PATCH /debug/trace_sampler handler. The GET path is wired separately in
+// MakeDebugHandler via the existing jsonHandler helper, matching the convention used by /debug/migrations and /debug/db/*.
+//
+// PATCH is necessarily bespoke because no other /debug/ endpoint takes a request body. It validates ratios in [0, 1] and
+// persists the change. The replica's in memory sampler picks up the change on the next poller tick (default 60s).
+func patchTraceSamplerHandler(logger *slog.Logger, ds fleet.Datastore) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// The /debug auth middleware always installs the viewer in context. If it is missing here, the middleware was bypassed
+		// and we should refuse to record a change rather than silently log user_id=0. That value is indistinguishable from a
+		// real user id of 0 and weakens the audit trail.
+		v, ok := viewer.FromContext(r.Context())
+		if !ok {
+			handleServerError(w, r, logger, "debug trace_sampler PATCH refused: viewer missing from context", "viewer required",
+				errors.New("viewer missing from context"))
+			return
+		}
+
+		var req traceSamplerPatchRequest
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+			http.Error(w, fmt.Sprintf("invalid JSON body: %v", err), http.StatusBadRequest)
+			return
+		}
+
+		if err := validateTraceSamplerPatch(req); err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		current, err := ds.GetTraceSamplerSettings(r.Context())
+		if err != nil {
+			handleServerError(w, r, logger, "debug trace_sampler PATCH read-modify failed", "internal error", err)
+			return
+		}
+
+		if req.HighVolumeRatio != nil {
+			current.HighVolumeRatio = *req.HighVolumeRatio
+		}
+		if req.StandardRatio != nil {
+			current.StandardRatio = *req.StandardRatio
+		}
+		if req.ForceFull != nil {
+			current.ForceFull = *req.ForceFull
+		}
+
+		if err := ds.SetTraceSamplerSettings(r.Context(), current); err != nil {
+			handleServerError(w, r, logger, "debug trace_sampler PATCH write failed", "internal error", err)
+			return
+		}
+
+		logger.InfoContext(r.Context(), "trace sampler settings updated",
+			"high_volume_ratio", current.HighVolumeRatio,
+			"standard_ratio", current.StandardRatio,
+			"force_full", current.ForceFull,
+			"updated_by_user_id", v.UserID(),
+		)
+
+		// Return the updated row so callers can confirm what was applied. Drop UpdatedAt: the row was read before the write,
+		// so current.UpdatedAt is the pre-write timestamp (stale and confusing). omitzero on the struct tag skips the field
+		// when zero. Operators who want the post-write timestamp can do a follow-up GET.
+		current.UpdatedAt = time.Time{}
+		b, err := json.MarshalIndent(current, "", "  ")
+		if err != nil {
+			handleServerError(w, r, logger, "debug trace_sampler PATCH encode response failed", "encoding response", err)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		_, _ = w.Write(b)
+	}
+}
+
+// handleServerError centralizes the internal-error response used throughout the trace sampler PATCH handler: it logs logMsg
+// with err, records err on the context for the error-handling middleware, and writes clientMsg to the client as a 500.
+func handleServerError(w http.ResponseWriter, r *http.Request, logger *slog.Logger, logMsg, clientMsg string, err error) {
+	logger.ErrorContext(r.Context(), logMsg, "err", err)
+	ctxerr.Handle(r.Context(), err)
+	http.Error(w, clientMsg, http.StatusInternalServerError)
+}
+
+func validateTraceSamplerPatch(req traceSamplerPatchRequest) error {
+	if req.HighVolumeRatio == nil && req.StandardRatio == nil && req.ForceFull == nil {
+		return errors.New("request body must include at least one of high_volume_ratio, standard_ratio, force_full")
+	}
+	if req.HighVolumeRatio != nil && (*req.HighVolumeRatio < 0 || *req.HighVolumeRatio > 1) {
+		return fmt.Errorf("high_volume_ratio must be in [0, 1], got %v", *req.HighVolumeRatio)
+	}
+	if req.StandardRatio != nil && (*req.StandardRatio < 0 || *req.StandardRatio > 1) {
+		return fmt.Errorf("standard_ratio must be in [0, 1], got %v", *req.StandardRatio)
+	}
+	return nil
+}

server/service/debug_trace_sampler_test.go

@@ -0,0 +1,234 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	mockds "github.com/fleetdm/fleet/v4/server/mock"
+	"github.com/fleetdm/fleet/v4/server/platform/tracing"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func discardLogger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(io.Discard, nil))
+}
+
+// adminAuthedRequest builds a request and primes the mockService so the debug auth middleware lets it through as a global
+// admin.
+func adminAuthedRequest(t *testing.T, method, target string, body string) (*mockService, *http.Request) {
+	t.Helper()
+	svc := &mockService{}
+	svc.On("GetSessionByKey", mock.Anything, "fake_session_key").
+		Return(&fleet.Session{UserID: 42, ID: 1}, nil)
+	svc.On("UserUnauthorized", mock.Anything, uint(42)).
+		Return(&fleet.User{ID: 42, GlobalRole: new(fleet.RoleAdmin)}, nil)
+
+	var reqBody io.Reader
+	if body != "" {
+		reqBody = strings.NewReader(body)
+	}
+	req := httptest.NewRequest(method, target, reqBody)
+	req.Header.Add("Authorization", "BEARER fake_session_key")
+	return svc, req
+}
+
+func TestTraceSamplerHandler_GET(t *testing.T) {
+	svc, req := adminAuthedRequest(t, http.MethodGet, "https://fleetdm.com/debug/trace_sampler", "")
+
+	ds := new(mockds.Store)
+	ds.GetTraceSamplerSettingsFunc = func(_ context.Context) (*tracing.Settings, error) {
+		return &tracing.Settings{
+			HighVolumeRatio: 0.001,
+			StandardRatio:   0.02,
+			ForceFull:       false,
+		}, nil
+	}
+
+	handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds)
+	res := httptest.NewRecorder()
+	handler.ServeHTTP(res, req)
+
+	require.Equal(t, http.StatusOK, res.Code)
+	require.True(t, ds.GetTraceSamplerSettingsFuncInvoked)
+
+	var got tracing.Settings
+	require.NoError(t, json.Unmarshal(res.Body.Bytes(), &got))
+	require.InDelta(t, 0.001, got.HighVolumeRatio, 1e-9)
+	require.InDelta(t, 0.02, got.StandardRatio, 1e-9)
+	require.False(t, got.ForceFull)
+}
+
+func TestTraceSamplerHandler_PATCH_PersistsChangesAndReturnsRow(t *testing.T) {
+	svc, req := adminAuthedRequest(t, http.MethodPatch,
+		"https://fleetdm.com/debug/trace_sampler",
+		`{"force_full": true}`)
+
+	ds := new(mockds.Store)
+	ds.GetTraceSamplerSettingsFunc = func(_ context.Context) (*tracing.Settings, error) {
+		return &tracing.Settings{
+			HighVolumeRatio: 0.001,
+			StandardRatio:   0.02,
+			ForceFull:       false,
+		}, nil
+	}
+	var saved *tracing.Settings
+	ds.SetTraceSamplerSettingsFunc = func(_ context.Context, s *tracing.Settings) error {
+		saved = s
+		return nil
+	}
+
+	handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds)
+	res := httptest.NewRecorder()
+	handler.ServeHTTP(res, req)
+
+	require.Equal(t, http.StatusOK, res.Code, "PATCH should return 200, body=%s", res.Body.String())
+	require.True(t, ds.SetTraceSamplerSettingsFuncInvoked)
+	require.NotNil(t, saved)
+	require.True(t, saved.ForceFull, "force_full should now be true")
+	require.InDelta(t, 0.001, saved.HighVolumeRatio, 1e-9, "other fields should be preserved")
+
+	// Verify the response body matches what was saved. If we forgot to write the response, the test would still see 200 from
+	// httptest's default but the body would be empty.
+	var returned tracing.Settings
+	require.NoError(t, json.Unmarshal(res.Body.Bytes(), &returned))
+	require.True(t, returned.ForceFull)
+	require.InDelta(t, saved.HighVolumeRatio, returned.HighVolumeRatio, 1e-9)
+	require.InDelta(t, saved.StandardRatio, returned.StandardRatio, 1e-9)
+
+	// PATCH response must NOT include updated_at. The handler reads the row before the write, so the pre-write timestamp
+	// would be stale. Operators do a follow-up GET to see the post-write value.
+	require.NotContains(t, res.Body.String(), "updated_at",
+		"PATCH response must drop updated_at to avoid returning a stale timestamp")
+}
+
+func TestTraceSamplerHandler_PATCH_PartialUpdatePreservesOtherFields(t *testing.T) {
+	// Locks in the docstring claim that "PATCH semantics mean only the provided fields are applied." Sending only
+	// high_volume_ratio must leave standard_ratio and force_full at their prior values.
+	svc, req := adminAuthedRequest(t, http.MethodPatch,
+		"https://fleetdm.com/debug/trace_sampler",
+		`{"high_volume_ratio": 0.5}`)
+
+	ds := new(mockds.Store)
+	ds.GetTraceSamplerSettingsFunc = func(_ context.Context) (*tracing.Settings, error) {
+		return &tracing.Settings{
+			HighVolumeRatio: 0.001,
+			StandardRatio:   0.07,
+			ForceFull:       true,
+		}, nil
+	}
+	var saved *tracing.Settings
+	ds.SetTraceSamplerSettingsFunc = func(_ context.Context, s *tracing.Settings) error {
+		saved = s
+		return nil
+	}
+
+	handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds)
+	res := httptest.NewRecorder()
+	handler.ServeHTTP(res, req)
+
+	require.Equal(t, http.StatusOK, res.Code, "body=%s", res.Body.String())
+	require.NotNil(t, saved)
+	require.InDelta(t, 0.5, saved.HighVolumeRatio, 1e-9, "high_volume_ratio should be applied")
+	require.InDelta(t, 0.07, saved.StandardRatio, 1e-9, "standard_ratio should be preserved from the prior row")
+	require.True(t, saved.ForceFull, "force_full should be preserved from the prior row")
+}
+
+func TestTraceSamplerHandler_PATCH_ReadFailureReturns500(t *testing.T) {
+	svc, req := adminAuthedRequest(t, http.MethodPatch,
+		"https://fleetdm.com/debug/trace_sampler",
+		`{"force_full": true}`)
+
+	ds := new(mockds.Store)
+	ds.GetTraceSamplerSettingsFunc = func(_ context.Context) (*tracing.Settings, error) {
+		return nil, errors.New("db unavailable")
+	}
+
+	handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds)
+	res := httptest.NewRecorder()
+	handler.ServeHTTP(res, req)
+
+	require.Equal(t, http.StatusInternalServerError, res.Code)
+	require.False(t, ds.SetTraceSamplerSettingsFuncInvoked, "should not attempt to write when read fails")
+}
+
+func TestTraceSamplerHandler_PATCH_WriteFailureReturns500(t *testing.T) {
+	svc, req := adminAuthedRequest(t, http.MethodPatch,
+		"https://fleetdm.com/debug/trace_sampler",
+		`{"force_full": true}`)
+
+	ds := new(mockds.Store)
+	ds.GetTraceSamplerSettingsFunc = func(_ context.Context) (*tracing.Settings, error) {
+		return &tracing.Settings{HighVolumeRatio: 0.001, StandardRatio: 0.02}, nil
+	}
+	ds.SetTraceSamplerSettingsFunc = func(_ context.Context, _ *tracing.Settings) error {
+		return errors.New("constraint violation")
+	}
+
+	handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds)
+	res := httptest.NewRecorder()
+	handler.ServeHTTP(res, req)
+
+	require.Equal(t, http.StatusInternalServerError, res.Code)
+	require.True(t, ds.SetTraceSamplerSettingsFuncInvoked)
+}
+
+func TestTraceSamplerHandler_PATCH_RejectsBadJSON(t *testing.T) {
+	svc, req := adminAuthedRequest(t, http.MethodPatch,
+		"https://fleetdm.com/debug/trace_sampler",
+		`{"force_full":`) // malformed
+
+	ds := new(mockds.Store)
+	handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds)
+	res := httptest.NewRecorder()
+	handler.ServeHTTP(res, req)
+
+	require.Equal(t, http.StatusBadRequest, res.Code)
+	require.False(t, ds.SetTraceSamplerSettingsFuncInvoked)
+}
+
+func TestTraceSamplerHandler_PATCH_RejectsOutOfRangeRatio(t *testing.T) {
+	cases := []struct {
+		name string
+		body string
+	}{
+		{"high above 1", `{"high_volume_ratio": 1.5}`},
+		{"high below 0", `{"high_volume_ratio": -0.1}`},
+		{"standard above 1", `{"standard_ratio": 2.0}`},
+		{"standard below 0", `{"standard_ratio": -1.0}`},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			svc, req := adminAuthedRequest(t, http.MethodPatch,
+				"https://fleetdm.com/debug/trace_sampler", c.body)
+			ds := new(mockds.Store)
+			handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds)
+			res := httptest.NewRecorder()
+			handler.ServeHTTP(res, req)
+
+			require.Equal(t, http.StatusBadRequest, res.Code)
+			require.Contains(t, res.Body.String(), "must be in [0, 1]")
+			require.False(t, ds.SetTraceSamplerSettingsFuncInvoked)
+		})
+	}
+}
+
+func TestTraceSamplerHandler_PATCH_RequiresAtLeastOneField(t *testing.T) {
+	svc, req := adminAuthedRequest(t, http.MethodPatch,
+		"https://fleetdm.com/debug/trace_sampler", `{}`)
+	ds := new(mockds.Store)
+	handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds)
+	res := httptest.NewRecorder()
+	handler.ServeHTTP(res, req)
+
+	require.Equal(t, http.StatusBadRequest, res.Code)
+	require.Contains(t, res.Body.String(), "at least one")
+}

server/service/tracing_tiers.go

@@ -0,0 +1,62 @@
+package service
+
+import (
+	"net/http"
+
+	"github.com/fleetdm/fleet/v4/server/platform/tracing"
+)
+
+// RegisterTracingTiers populates the trace sampling registry with the tier classifications for routes owned by this package
+// (the legacy flat server/service handler).
+//
+// Bounded contexts that have moved out of this package (e.g. server/activity/) own their own registrations. They
+// should expose a RegisterTracingTiers function the serve command calls during startup. The platform/tracing package itself
+// stays free of route knowledge.
+//
+// Paths use "_version_" as the placeholder for the gorilla/mux fleetversion segment. The sampler normalizes incoming span
+// names back to that form before lookup. Alternate path forms (e.g. /api/v1/osquery/...) are registered separately.
+//
+// Unregistered routes (including all cron jobs, enroll, SCEP, MDM checkin, command ack/result, GitOps batch, etc.) fall to
+// TierAlways by design.
+func RegisterTracingTiers(registry *tracing.Registry) {
+	// Hot agent endpoints. These dominate request volume at scale (tens of thousands of spans per second on a 100k-host
+	// fleet) without being individually interesting. Sampled at the configured high volume ratio (default 0.1%).
+	registry.Register(http.MethodPost, "/api/osquery/config", tracing.TierHighVolume)
+	registry.Register(http.MethodPost, "/api/v1/osquery/config", tracing.TierHighVolume)
+	registry.Register(http.MethodPost, "/api/osquery/distributed/read", tracing.TierHighVolume)
+	registry.Register(http.MethodPost, "/api/v1/osquery/distributed/read", tracing.TierHighVolume)
+	registry.Register(http.MethodPost, "/api/osquery/distributed/write", tracing.TierHighVolume)
+	registry.Register(http.MethodPost, "/api/v1/osquery/distributed/write", tracing.TierHighVolume)
+	registry.Register(http.MethodPost, "/api/osquery/log", tracing.TierHighVolume)
+	registry.Register(http.MethodPost, "/api/v1/osquery/log", tracing.TierHighVolume)
+	registry.Register(http.MethodPost, "/api/fleet/orbit/config", tracing.TierHighVolume)
+	registry.Register(http.MethodPost, "/api/fleet/orbit/device_token", tracing.TierHighVolume)
+	registry.Register(http.MethodHead, "/api/fleet/orbit/ping", tracing.TierHighVolume)
+	registry.Register(http.MethodHead, "/api/fleet/device/ping", tracing.TierHighVolume)
+	registry.Register(http.MethodHead, "/api/_version_/fleet/device/{token}/ping", tracing.TierHighVolume)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/device/{token}/desktop", tracing.TierHighVolume)
+
+	// Admin / read endpoints. Moderate volume, moderate diagnostic value. Sampled at the configured standard ratio (default
+	// 2%). Starting set covers the highest traffic admin reads plus the per page load endpoints (config, me, version, etc.)
+	// that the UI hits on every navigation. Expand as we observe traffic patterns in dogfood and the customer pilot.
+	registry.Register(http.MethodGet, "/api/_version_/fleet/config", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/fleets", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/host_summary", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/hosts", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/hosts/{id}", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/hosts/count", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/hosts/identifier/{identifier}", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/hosts/summary/mdm", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/labels", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/me", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/policies", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/reports", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/software", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/software/titles", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/software/versions", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/spec/enroll_secret", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/users", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/version", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/charts/{metric}", tracing.TierStandard)
+	registry.Register(http.MethodGet, "/api/_version_/fleet/android_enterprise", tracing.TierStandard)
+}