From 59a673bc15f53f5aad54898025e9441fdfaf4efc Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky <2685025+getvictor@users.noreply.github.com> Date: Tue, 2 Jun 2026 18:53:00 -0500 Subject: [PATCH] Added trace sampler to use OTEL in prod. (#46595) **Related issue:** Resolves #44652 Docs: https://github.com/fleetdm/fleet/pull/46631 # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files) for more information. - [x] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements), JS inline code is prevented especially for url redirects, and untrusted data interpolated into shell scripts/commands is validated against shell metacharacters. ## Testing - [x] Added/updated automated tests - [x] QA'd all new/changed functionality manually ## Database migrations - [x] Ensured the correct collation is explicitly set for character columns (`COLLATE utf8mb4_unicode_ci`). ## New Fleet configuration settings - [x] Setting(s) is/are explicitly excluded from GitOps ## Summary by CodeRabbit * **New Features** * Route-aware OpenTelemetry trace sampling with tiered default ratios (very low for select high-volume routes, reduced rate for admin reads, full sampling otherwise). * Admin-only GET/PATCH /debug/trace_sampler to view and update sampling ratios and a runtime "force full" toggle. * Liveness probe endpoints (/healthz, /version, /metrics) are excluded from tracing; settings propagate to replicas at runtime without restart. --- changes/44652-trace-sampler | 2 + cmd/fleet/otel.go | 59 +++-- cmd/fleet/otel_test.go | 10 +- cmd/fleet/serve.go | 27 +- server/activity/bootstrap/bootstrap.go | 6 + server/activity/internal/service/handler.go | 11 + .../20260601200727_AddTraceSamplerSettings.go | 39 +++ server/datastore/mysql/schema.sql | 19 +- server/datastore/mysql/trace_sampler.go | 56 +++++ server/datastore/mysql/trace_sampler_test.go | 65 +++++ server/fleet/datastore.go | 11 + server/mock/datastore_mock.go | 25 ++ server/platform/tracing/doc.go | 13 + server/platform/tracing/registry.go | 92 +++++++ server/platform/tracing/registry_test.go | 176 +++++++++++++ server/platform/tracing/sampler.go | 105 ++++++++ server/platform/tracing/sampler_test.go | 200 +++++++++++++++ server/platform/tracing/settings.go | 13 + server/platform/tracing/settings_poller.go | 72 ++++++ .../platform/tracing/settings_poller_test.go | 139 +++++++++++ server/service/debug_handler.go | 9 +- server/service/debug_trace_sampler.go | 113 +++++++++ server/service/debug_trace_sampler_test.go | 234 ++++++++++++++++++ server/service/tracing_tiers.go | 62 +++++ 24 files changed, 1529 insertions(+), 29 deletions(-) create mode 100644 changes/44652-trace-sampler create mode 100644 server/datastore/mysql/migrations/tables/20260601200727_AddTraceSamplerSettings.go create mode 100644 server/datastore/mysql/trace_sampler.go create mode 100644 server/datastore/mysql/trace_sampler_test.go create mode 100644 server/platform/tracing/doc.go create mode 100644 server/platform/tracing/registry.go create mode 100644 server/platform/tracing/registry_test.go create mode 100644 server/platform/tracing/sampler.go create mode 100644 server/platform/tracing/sampler_test.go create mode 100644 server/platform/tracing/settings.go create mode 100644 server/platform/tracing/settings_poller.go create mode 100644 server/platform/tracing/settings_poller_test.go create mode 100644 server/service/debug_trace_sampler.go create mode 100644 server/service/debug_trace_sampler_test.go create mode 100644 server/service/tracing_tiers.go diff --git a/changes/44652-trace-sampler b/changes/44652-trace-sampler new file mode 100644 index 0000000000..471a000dc0 --- /dev/null +++ b/changes/44652-trace-sampler @@ -0,0 +1,2 @@ +- Added route-aware head sampling for OpenTelemetry trace export. When `tracing_enabled` is on, agent firehose endpoints (osquery distributed read/write, orbit ping/config, device desktop/ping) are sampled at 0.1% by default, admin reads at 2%, and everything else (enroll, SCEP, MDM checkin, cron jobs, GitOps batch) at 100%. Liveness probes (`/healthz`, `/version`, `/metrics`) are dropped unconditionally. +- Added `GET`/`PATCH /debug/trace_sampler` (admin only, behind the existing `/debug` auth) for adjusting ratios or flipping a 100% `force_full` debug window at runtime. Each Fleet replica polls the new `trace_sampler_settings` row every 60 seconds and applies changes without a restart. diff --git a/cmd/fleet/otel.go b/cmd/fleet/otel.go index e0defd8ceb..72668a6428 100644 --- a/cmd/fleet/otel.go +++ b/cmd/fleet/otel.go @@ -4,6 +4,7 @@ import ( "context" "github.com/fleetdm/fleet/v4/server/config" + "github.com/fleetdm/fleet/v4/server/platform/tracing" "github.com/fleetdm/fleet/v4/server/version" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" @@ -17,26 +18,25 @@ import ( semconv "go.opentelemetry.io/otel/semconv/v1.40.0" ) -// initOTELProviders constructs the OpenTelemetry trace, metric, and (when -// log export is enabled) log providers. Returns nil providers when OTEL is -// disabled in the configuration. Fatal errors during exporter setup are -// reported through initFatal so the server can fail fast at startup. +// initOTELProviders constructs the OpenTelemetry trace, metric, and (when log export is enabled) log providers. Returns nil +// providers when OTEL is disabled in the configuration. Fatal errors during exporter setup are reported through initFatal so +// the server can fail fast at startup. // -// As a side effect, the constructed tracer and meter providers are registered -// as the OTEL globals via otel.SetTracerProvider and otel.SetMeterProvider — -// matching the original inline behavior in runServeCmd. -func initOTELProviders(cfg config.FleetConfig, initFatal func(err error, msg string)) ( +// The traceRegistry is consulted on every sampling decision. Populating it (with route to tier mappings) is the caller's +// responsibility. The canonical pattern is for each bounded context to register its own routes at startup. See +// server/service/tracing_tiers.go for the legacy non modularized routes. +func initOTELProviders(cfg config.FleetConfig, traceRegistry *tracing.Registry, initFatal func(err error, msg string)) ( *otelsdklog.LoggerProvider, *sdktrace.TracerProvider, *sdkmetric.MeterProvider, + *tracing.RouteTierSampler, ) { if !cfg.OTELEnabled() { - return nil, nil, nil + return nil, nil, nil, nil } - // Create shared resource with service identification attributes. - // OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES env vars can override - // the defaults below. + // Create shared resource with service identification attributes. OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES env vars + // can override the defaults below. res, err := resource.New(context.Background(), resource.WithSchemaURL(semconv.SchemaURL), resource.WithAttributes( @@ -48,9 +48,8 @@ func initOTELProviders(cfg config.FleetConfig, initFatal func(err error, msg str ) if err != nil { initFatal(err, "Failed to create OTEL resource") - // Returning here makes the function safe even if a caller's - // initFatal does not terminate (e.g., tests using a recorder). - return nil, nil, nil + // Returning here makes the function safe even if a caller's initFatal does not terminate (e.g. tests using a recorder). + return nil, nil, nil, nil } // Initialize OTEL traces. @@ -59,16 +58,34 @@ func initOTELProviders(cfg config.FleetConfig, initFatal func(err error, msg str )) if err != nil { initFatal(err, "Failed to initialize OTEL trace exporter") - return nil, nil, nil + return nil, nil, nil, nil } - // Configure batch span processor with smaller batch size to avoid exceeding - // message size limits (4MB default limit). + // Configure batch span processor with smaller batch size to avoid exceeding message size limits (4MB default limit). batchSpanProcessor := sdktrace.NewBatchSpanProcessor(otlpTraceExporter, sdktrace.WithMaxExportBatchSize(256), // Reduce from default 512 to 256 ) + // Route aware head sampler. The sampler starts with compile time defaults. The settings poller re-reads trace_sampler_settings + // every 60s and calls Apply on change. We wrap with ParentBased so a remote sampled parent (e.g. an upstream span) keeps the + // whole trace coherent even on the hot agent path. + sampler := tracing.NewRouteTierSampler(traceRegistry) + // ParentBased semantics: + // + // Remote parent SAMPLED: honor the upstream decision via AlwaysSample. + // + // Remote parent NOT SAMPLED: honor it via NeverSample, keeping trace coherence across services. Defaulting to our local + // sampler here would let us locally sample a span whose upstream parent was explicitly dropped, breaking distributed + // traces. + // + // No remote parent: use our route aware sampler. + parentBased := sdktrace.ParentBased( + sampler, + sdktrace.WithRemoteParentSampled(sdktrace.AlwaysSample()), + sdktrace.WithRemoteParentNotSampled(sdktrace.NeverSample()), + ) tracerProvider := sdktrace.NewTracerProvider( sdktrace.WithResource(res), sdktrace.WithSpanProcessor(batchSpanProcessor), + sdktrace.WithSampler(parentBased), ) otel.SetTracerProvider(tracerProvider) @@ -78,7 +95,7 @@ func initOTELProviders(cfg config.FleetConfig, initFatal func(err error, msg str ) if err != nil { initFatal(err, "Failed to initialize OTEL metrics exporter") - return nil, nil, nil + return nil, nil, nil, nil } // Create views to rename otelsql metrics to match what OpenTelemetry Signoz expects. @@ -125,7 +142,7 @@ func initOTELProviders(cfg config.FleetConfig, initFatal func(err error, msg str ) if err != nil { initFatal(err, "Failed to initialize OTEL log exporter") - return nil, nil, nil + return nil, nil, nil, nil } loggerProvider = otelsdklog.NewLoggerProvider( otelsdklog.WithResource(res), @@ -133,5 +150,5 @@ func initOTELProviders(cfg config.FleetConfig, initFatal func(err error, msg str ) } - return loggerProvider, tracerProvider, meterProvider + return loggerProvider, tracerProvider, meterProvider, sampler } diff --git a/cmd/fleet/otel_test.go b/cmd/fleet/otel_test.go index 3f5c73e5e8..7b75276120 100644 --- a/cmd/fleet/otel_test.go +++ b/cmd/fleet/otel_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/fleetdm/fleet/v4/server/config" + "github.com/fleetdm/fleet/v4/server/platform/tracing" otelsdklog "go.opentelemetry.io/otel/sdk/log" sdkmetric "go.opentelemetry.io/otel/sdk/metric" sdktrace "go.opentelemetry.io/otel/sdk/trace" @@ -41,12 +42,13 @@ func TestInitOTELProviders_DisabledReturnsNilProviders(t *testing.T) { require.False(t, cfg.OTELEnabled(), "precondition: default config must have OTEL disabled") called := false - lp, tp, mp := initOTELProviders(cfg, func(err error, msg string) { called = true }) + lp, tp, mp, sampler := initOTELProviders(cfg, tracing.NewRegistry(), func(err error, msg string) { called = true }) require.False(t, called, "initFatal must not be called when OTEL is disabled") require.Nil(t, lp) require.Nil(t, tp) require.Nil(t, mp) + require.Nil(t, sampler, "sampler must be nil when OTEL is disabled") } func TestInitOTELProviders_EnabledReturnsTracerAndMeterProviders(t *testing.T) { @@ -60,13 +62,14 @@ func TestInitOTELProviders_EnabledReturnsTracerAndMeterProviders(t *testing.T) { require.True(t, cfg.OTELEnabled(), "precondition: tracing-enabled config must have OTEL enabled") called := false - lp, tp, mp := initOTELProviders(cfg, func(err error, msg string) { called = true }) + lp, tp, mp, sampler := initOTELProviders(cfg, tracing.NewRegistry(), func(err error, msg string) { called = true }) shutdownOTELProviders(t, lp, tp, mp) require.False(t, called, "initFatal must not be called for a healthy enabled config") require.Nil(t, lp, "logger provider should be nil when OtelLogsEnabled is false") require.NotNil(t, tp) require.NotNil(t, mp) + require.NotNil(t, sampler, "sampler must be returned when OTEL is enabled") } func TestInitOTELProviders_LogExportEnabledReturnsLoggerProvider(t *testing.T) { @@ -75,11 +78,12 @@ func TestInitOTELProviders_LogExportEnabledReturnsLoggerProvider(t *testing.T) { } called := false - lp, tp, mp := initOTELProviders(cfg, func(err error, msg string) { called = true }) + lp, tp, mp, sampler := initOTELProviders(cfg, tracing.NewRegistry(), func(err error, msg string) { called = true }) shutdownOTELProviders(t, lp, tp, mp) require.False(t, called) require.NotNil(t, lp, "logger provider should be set when OtelLogsEnabled is true") require.NotNil(t, tp) require.NotNil(t, mp) + require.NotNil(t, sampler) } diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 7203808c8f..4eb08fccb5 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -85,6 +85,7 @@ import ( platform_http "github.com/fleetdm/fleet/v4/server/platform/http" platform_logging "github.com/fleetdm/fleet/v4/server/platform/logging" common_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql" + "github.com/fleetdm/fleet/v4/server/platform/tracing" "github.com/fleetdm/fleet/v4/server/pubsub" "github.com/fleetdm/fleet/v4/server/service" "github.com/fleetdm/fleet/v4/server/service/async" @@ -181,8 +182,16 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev // Validate OTEL server options config.Logging.Validate(initFatal) - // Init OTEL providers (traces, metrics, logs) - loggerProvider, tracerProvider, meterProvider := initOTELProviders(config, initFatal) + // Trace sampler tier registry. Bounded contexts register their own route to tier classifications below. The + // platform/tracing package stays free of cross context coupling. Infra paths (/healthz, /version, /metrics) are registered + // alongside their WrapHandler calls further down in this function. + traceRegistry := tracing.NewRegistry() + service.RegisterTracingTiers(traceRegistry) + activity_bootstrap.RegisterTracingTiers(traceRegistry) + // Future bounded contexts: each exposes its own RegisterTracingTiers. + + // Init OTEL providers (traces, metrics, logs) and the route aware sampler. + loggerProvider, tracerProvider, meterProvider, traceSampler := initOTELProviders(config, traceRegistry, initFatal) logger := initLogger(config, loggerProvider) @@ -1037,6 +1046,13 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev } }() + // Trace sampler runtime control. The poller re-reads trace_sampler_settings every 60s and atomically swaps the sampler's + // ratios and force_full so support can flip a 100% debug window via PATCH /debug/trace_sampler without restarting any + // replicas. No-op when OTEL is disabled. + if traceSampler != nil { + go tracing.StartSettingsPoller(ctx, traceSampler, ds, logger) + } + if softwareInstallStore != nil { if err := cronSchedules.StartCronSchedule( func() (fleet.CronSchedule, error) { @@ -1474,6 +1490,13 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev ), healthCheckers) rootMux := http.NewServeMux() + // Infra paths (liveness, version, metrics) are platform owned and have zero diagnostic value as traces once their metric + // counterparts exist. Drop them unconditionally, even under ForceFull, since a 100% debug window should not flood SigNoz + // with probe spans. Register /metrics here even though its handler is mounted conditionally below. Registering a route + // whose handler isn't installed is a harmless lookup table entry and keeps the policy in one place. + traceRegistry.Register(http.MethodGet, "/healthz", tracing.TierNever) + traceRegistry.Register(http.MethodGet, "/version", tracing.TierNever) + traceRegistry.Register(http.MethodGet, "/metrics", tracing.TierNever) rootMux.Handle("/healthz", service.PrometheusMetricsHandler("healthz", otelmw.WrapHandler(health.Handler(httpLogger, healthCheckers), "/healthz", config))) rootMux.Handle("/version", service.PrometheusMetricsHandler("version", otelmw.WrapHandler(version.Handler(), "/version", config))) rootMux.Handle("/assets/", service.PrometheusMetricsHandler("static_assets", otelmw.WrapHandlerDynamic(service.ServeStaticAssets("/assets/", serveCSP), config))) diff --git a/server/activity/bootstrap/bootstrap.go b/server/activity/bootstrap/bootstrap.go index 176a2da7ab..684c8fd301 100644 --- a/server/activity/bootstrap/bootstrap.go +++ b/server/activity/bootstrap/bootstrap.go @@ -12,6 +12,7 @@ import ( platform_authz "github.com/fleetdm/fleet/v4/server/platform/authz" eu "github.com/fleetdm/fleet/v4/server/platform/endpointer" platform_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql" + "github.com/fleetdm/fleet/v4/server/platform/tracing" "github.com/go-kit/kit/endpoint" ) @@ -31,3 +32,8 @@ func New( return svc, routesFn } + +// RegisterTracingTiers classifies the activity context's routes for trace sampling. +func RegisterTracingTiers(registry *tracing.Registry) { + service.RegisterTracingTiers(registry) +} diff --git a/server/activity/internal/service/handler.go b/server/activity/internal/service/handler.go index a5f1c391db..c1789e0d0b 100644 --- a/server/activity/internal/service/handler.go +++ b/server/activity/internal/service/handler.go @@ -2,11 +2,13 @@ package service import ( "context" + "net/http" "github.com/fleetdm/fleet/v4/server/activity/api" api_http "github.com/fleetdm/fleet/v4/server/activity/api/http" eu "github.com/fleetdm/fleet/v4/server/platform/endpointer" platform_http "github.com/fleetdm/fleet/v4/server/platform/http" + "github.com/fleetdm/fleet/v4/server/platform/tracing" "github.com/go-kit/kit/endpoint" kithttp "github.com/go-kit/kit/transport/http" "github.com/gorilla/mux" @@ -27,6 +29,15 @@ func attachFleetAPIRoutes(r *mux.Router, svc api.Service, authMiddleware endpoin ue.GET("/api/_version_/fleet/hosts/{id:[0-9]+}/activities", listHostPastActivitiesEndpoint, api_http.ListHostPastActivitiesRequest{}) } +// RegisterTracingTiers classifies this context's routes for trace sampling. Both activity list endpoints are admin reads, so +// they belong in the standard tier (default 2%). Kept next to the route registrations above so the two stay in sync. The +// sampler's version normalizer collapses the {fleetversion:...} segment, so the "_version_" placeholder matches the rendered +// span name regardless of the configured API versions. +func RegisterTracingTiers(registry *tracing.Registry) { + registry.Register(http.MethodGet, "/api/_version_/fleet/activities", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/hosts/{id}/activities", tracing.TierStandard) +} + func apiVersions() []string { return []string{"v1", "latest"} } diff --git a/server/datastore/mysql/migrations/tables/20260601200727_AddTraceSamplerSettings.go b/server/datastore/mysql/migrations/tables/20260601200727_AddTraceSamplerSettings.go new file mode 100644 index 0000000000..ca8454feeb --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20260601200727_AddTraceSamplerSettings.go @@ -0,0 +1,39 @@ +package tables + +import ( + "database/sql" +) + +func init() { + MigrationClient.AddMigration(Up_20260601200727, Down_20260601200727) +} + +func Up_20260601200727(tx *sql.Tx) error { + // Singleton settings row holding the runtime tunable trace sampling configuration. Operators flip these via PATCH + // /debug/trace_sampler. The /debug auth log and the PATCH access log already record who made the change. + _, err := tx.Exec(` + CREATE TABLE trace_sampler_settings ( + id TINYINT UNSIGNED NOT NULL PRIMARY KEY, + high_volume_ratio DOUBLE NOT NULL DEFAULT 0.001, + standard_ratio DOUBLE NOT NULL DEFAULT 0.02, + force_full TINYINT(1) NOT NULL DEFAULT 0, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT ck_trace_sampler_settings_singleton CHECK (id = 1), + CONSTRAINT ck_trace_sampler_settings_high_range CHECK (high_volume_ratio BETWEEN 0 AND 1), + CONSTRAINT ck_trace_sampler_settings_std_range CHECK (standard_ratio BETWEEN 0 AND 1) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + `) + if err != nil { + return err + } + + // Insert the seed row with an explicit fixed updated_at instead of letting the column default to CURRENT_TIMESTAMP. + // schema.sql captures the row's data; if updated_at varied per migration run, every developer would get a different schema + // dump and `make test-schema` would never pass. + _, err = tx.Exec(`INSERT INTO trace_sampler_settings (id, updated_at) VALUES (1, '2026-06-01 00:00:00')`) + return err +} + +func Down_20260601200727(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 3cb63b42d0..fc8bbbe06a 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -2011,9 +2011,9 @@ CREATE TABLE `migration_status_tables` ( `is_applied` tinyint(1) NOT NULL, `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=537 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=538 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'),(350,20250124194347,1,'2020-01-01 01:01:01'),(351,20250127162751,1,'2020-01-01 01:01:01'),(352,20250213104005,1,'2020-01-01 01:01:01'),(353,20250214205657,1,'2020-01-01 01:01:01'),(354,20250217093329,1,'2020-01-01 01:01:01'),(355,20250219090511,1,'2020-01-01 01:01:01'),(356,20250219100000,1,'2020-01-01 01:01:01'),(357,20250219142401,1,'2020-01-01 01:01:01'),(358,20250224184002,1,'2020-01-01 01:01:01'),(359,20250225085436,1,'2020-01-01 01:01:01'),(360,20250226000000,1,'2020-01-01 01:01:01'),(361,20250226153445,1,'2020-01-01 01:01:01'),(362,20250304162702,1,'2020-01-01 01:01:01'),(363,20250306144233,1,'2020-01-01 01:01:01'),(364,20250313163430,1,'2020-01-01 01:01:01'),(365,20250317130944,1,'2020-01-01 01:01:01'),(366,20250318165922,1,'2020-01-01 01:01:01'),(367,20250320132525,1,'2020-01-01 01:01:01'),(368,20250320200000,1,'2020-01-01 01:01:01'),(369,20250326161930,1,'2020-01-01 01:01:01'),(370,20250326161931,1,'2020-01-01 01:01:01'),(371,20250331042354,1,'2020-01-01 01:01:01'),(372,20250331154206,1,'2020-01-01 01:01:01'),(373,20250401155831,1,'2020-01-01 01:01:01'),(374,20250408133233,1,'2020-01-01 01:01:01'),(375,20250410104321,1,'2020-01-01 01:01:01'),(376,20250421085116,1,'2020-01-01 01:01:01'),(377,20250422095806,1,'2020-01-01 01:01:01'),(378,20250424153059,1,'2020-01-01 01:01:01'),(379,20250430103833,1,'2020-01-01 01:01:01'),(380,20250430112622,1,'2020-01-01 01:01:01'),(381,20250501162727,1,'2020-01-01 01:01:01'),(382,20250502154517,1,'2020-01-01 01:01:01'),(383,20250502222222,1,'2020-01-01 01:01:01'),(384,20250507170845,1,'2020-01-01 01:01:01'),(385,20250513162912,1,'2020-01-01 01:01:01'),(386,20250519161614,1,'2020-01-01 01:01:01'),(387,20250519170000,1,'2020-01-01 01:01:01'),(388,20250520153848,1,'2020-01-01 01:01:01'),(389,20250528115932,1,'2020-01-01 01:01:01'),(390,20250529102706,1,'2020-01-01 01:01:01'),(391,20250603105558,1,'2020-01-01 01:01:01'),(392,20250609102714,1,'2020-01-01 01:01:01'),(393,20250609112613,1,'2020-01-01 01:01:01'),(394,20250613103810,1,'2020-01-01 01:01:01'),(395,20250616193950,1,'2020-01-01 01:01:01'),(396,20250624140757,1,'2020-01-01 01:01:01'),(397,20250626130239,1,'2020-01-01 01:01:01'),(398,20250629131032,1,'2020-01-01 01:01:01'),(399,20250701155654,1,'2020-01-01 01:01:01'),(400,20250707095725,1,'2020-01-01 01:01:01'),(401,20250716152435,1,'2020-01-01 01:01:01'),(402,20250718091828,1,'2020-01-01 01:01:01'),(403,20250728122229,1,'2020-01-01 01:01:01'),(404,20250731122715,1,'2020-01-01 01:01:01'),(405,20250731151000,1,'2020-01-01 01:01:01'),(406,20250803000000,1,'2020-01-01 01:01:01'),(407,20250805083116,1,'2020-01-01 01:01:01'),(408,20250807140441,1,'2020-01-01 01:01:01'),(409,20250808000000,1,'2020-01-01 01:01:01'),(410,20250811155036,1,'2020-01-01 01:01:01'),(411,20250813205039,1,'2020-01-01 01:01:01'),(412,20250814123333,1,'2020-01-01 01:01:01'),(413,20250815130115,1,'2020-01-01 01:01:01'),(414,20250816115553,1,'2020-01-01 01:01:01'),(415,20250817154557,1,'2020-01-01 01:01:01'),(416,20250825113751,1,'2020-01-01 01:01:01'),(417,20250827113140,1,'2020-01-01 01:01:01'),(418,20250828120836,1,'2020-01-01 01:01:01'),(419,20250902112642,1,'2020-01-01 01:01:01'),(420,20250904091745,1,'2020-01-01 01:01:01'),(421,20250905090000,1,'2020-01-01 01:01:01'),(422,20250922083056,1,'2020-01-01 01:01:01'),(423,20250923120000,1,'2020-01-01 01:01:01'),(424,20250926123048,1,'2020-01-01 01:01:01'),(425,20251015103505,1,'2020-01-01 01:01:01'),(426,20251015103600,1,'2020-01-01 01:01:01'),(427,20251015103700,1,'2020-01-01 01:01:01'),(428,20251015103800,1,'2020-01-01 01:01:01'),(429,20251015103900,1,'2020-01-01 01:01:01'),(430,20251028140000,1,'2020-01-01 01:01:01'),(431,20251028140100,1,'2020-01-01 01:01:01'),(432,20251028140110,1,'2020-01-01 01:01:01'),(433,20251028140200,1,'2020-01-01 01:01:01'),(434,20251028140300,1,'2020-01-01 01:01:01'),(435,20251028140400,1,'2020-01-01 01:01:01'),(436,20251031154558,1,'2020-01-01 01:01:01'),(437,20251103160848,1,'2020-01-01 01:01:01'),(438,20251104112849,1,'2020-01-01 01:01:01'),(439,20251106000000,1,'2020-01-01 01:01:01'),(440,20251107164629,1,'2020-01-01 01:01:01'),(441,20251107170854,1,'2020-01-01 01:01:01'),(442,20251110172137,1,'2020-01-01 01:01:01'),(443,20251111153133,1,'2020-01-01 01:01:01'),(444,20251117020000,1,'2020-01-01 01:01:01'),(445,20251117020100,1,'2020-01-01 01:01:01'),(446,20251117020200,1,'2020-01-01 01:01:01'),(447,20251121100000,1,'2020-01-01 01:01:01'),(448,20251121124239,1,'2020-01-01 01:01:01'),(449,20251124090450,1,'2020-01-01 01:01:01'),(450,20251124135808,1,'2020-01-01 01:01:01'),(451,20251124140138,1,'2020-01-01 01:01:01'),(452,20251124162948,1,'2020-01-01 01:01:01'),(453,20251127113559,1,'2020-01-01 01:01:01'),(454,20251202162232,1,'2020-01-01 01:01:01'),(455,20251203170808,1,'2020-01-01 01:01:01'),(456,20251207050413,1,'2020-01-01 01:01:01'),(457,20251208215800,1,'2020-01-01 01:01:01'),(458,20251209221730,1,'2020-01-01 01:01:01'),(459,20251209221850,1,'2020-01-01 01:01:01'),(460,20251215163721,1,'2020-01-01 01:01:01'),(461,20251217000000,1,'2020-01-01 01:01:01'),(462,20251217120000,1,'2020-01-01 01:01:01'),(463,20251229000000,1,'2020-01-01 01:01:01'),(464,20251229000010,1,'2020-01-01 01:01:01'),(465,20251229000020,1,'2020-01-01 01:01:01'),(466,20260106000000,1,'2020-01-01 01:01:01'),(467,20260108200708,1,'2020-01-01 01:01:01'),(468,20260108214732,1,'2020-01-01 01:01:01'),(469,20260109231821,1,'2020-01-01 01:01:01'),(470,20260113012054,1,'2020-01-01 01:01:01'),(471,20260124200020,1,'2020-01-01 01:01:01'),(472,20260126150840,1,'2020-01-01 01:01:01'),(473,20260126210724,1,'2020-01-01 01:01:01'),(474,20260202151756,1,'2020-01-01 01:01:01'),(475,20260205184907,1,'2020-01-01 01:01:01'),(476,20260210151544,1,'2020-01-01 01:01:01'),(477,20260210155109,1,'2020-01-01 01:01:01'),(478,20260210181120,1,'2020-01-01 01:01:01'),(479,20260211200153,1,'2020-01-01 01:01:01'),(480,20260217141240,1,'2020-01-01 01:01:01'),(481,20260217200906,1,'2020-01-01 01:01:01'),(482,20260218175704,1,'2020-01-01 01:01:01'),(483,20260314120000,1,'2020-01-01 01:01:01'),(484,20260316120000,1,'2020-01-01 01:01:01'),(485,20260316120001,1,'2020-01-01 01:01:01'),(486,20260316120002,1,'2020-01-01 01:01:01'),(487,20260316120003,1,'2020-01-01 01:01:01'),(488,20260316120004,1,'2020-01-01 01:01:01'),(489,20260316120005,1,'2020-01-01 01:01:01'),(490,20260316120006,1,'2020-01-01 01:01:01'),(491,20260316120007,1,'2020-01-01 01:01:01'),(492,20260316120008,1,'2020-01-01 01:01:01'),(493,20260316120009,1,'2020-01-01 01:01:01'),(494,20260316120010,1,'2020-01-01 01:01:01'),(495,20260317120000,1,'2020-01-01 01:01:01'),(496,20260318184559,1,'2020-01-01 01:01:01'),(497,20260319120000,1,'2020-01-01 01:01:01'),(498,20260323144117,1,'2020-01-01 01:01:01'),(499,20260324161944,1,'2020-01-01 01:01:01'),(500,20260324223334,1,'2020-01-01 01:01:01'),(501,20260326131501,1,'2020-01-01 01:01:01'),(502,20260326210603,1,'2020-01-01 01:01:01'),(503,20260331000000,1,'2020-01-01 01:01:01'),(504,20260401153000,1,'2020-01-01 01:01:01'),(505,20260401153001,1,'2020-01-01 01:01:01'),(506,20260401153503,1,'2020-01-01 01:01:01'),(507,20260403120000,1,'2020-01-01 01:01:01'),(508,20260409153713,1,'2020-01-01 01:01:01'),(509,20260409153714,1,'2020-01-01 01:01:01'),(510,20260409153715,1,'2020-01-01 01:01:01'),(511,20260409153716,1,'2020-01-01 01:01:01'),(512,20260409153717,1,'2020-01-01 01:01:01'),(513,20260409183610,1,'2020-01-01 01:01:01'),(514,20260410173222,1,'2020-01-01 01:01:01'),(515,20260422181702,1,'2020-01-01 01:01:01'),(516,20260423161823,1,'2020-01-01 01:01:01'),(517,20260423161824,1,'2020-01-01 01:01:01'),(518,20260518194422,1,'2020-01-01 01:01:01'),(519,20260522195224,1,'2020-01-01 01:01:01'),(520,20260522195225,1,'2020-01-01 01:01:01'),(521,20260522195226,1,'2020-01-01 01:01:01'),(522,20260522195227,1,'2020-01-01 01:01:01'),(523,20260522195229,1,'2020-01-01 01:01:01'),(524,20260522195230,1,'2020-01-01 01:01:01'),(525,20260522195231,1,'2020-01-01 01:01:01'),(526,20260522195232,1,'2020-01-01 01:01:01'),(527,20260522195233,1,'2020-01-01 01:01:01'),(528,20260522195234,1,'2020-01-01 01:01:01'),(529,20260522195235,1,'2020-01-01 01:01:01'),(530,20260527215817,1,'2020-01-01 01:01:01'),(531,20260528201143,1,'2020-01-01 01:01:01'),(532,20260528201150,1,'2020-01-01 01:01:01'),(533,20260528211626,1,'2020-01-01 01:01:01'),(534,20260528213326,1,'2020-01-01 01:01:01'),(535,20260529091823,1,'2020-01-01 01:01:01'),(536,20260529120000,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241220114904,1,'2020-01-01 01:01:01'),(342,20241224000000,1,'2020-01-01 01:01:01'),(343,20241230000000,1,'2020-01-01 01:01:01'),(344,20241231112624,1,'2020-01-01 01:01:01'),(345,20250102121439,1,'2020-01-01 01:01:01'),(346,20250121094045,1,'2020-01-01 01:01:01'),(347,20250121094500,1,'2020-01-01 01:01:01'),(348,20250121094600,1,'2020-01-01 01:01:01'),(349,20250121094700,1,'2020-01-01 01:01:01'),(350,20250124194347,1,'2020-01-01 01:01:01'),(351,20250127162751,1,'2020-01-01 01:01:01'),(352,20250213104005,1,'2020-01-01 01:01:01'),(353,20250214205657,1,'2020-01-01 01:01:01'),(354,20250217093329,1,'2020-01-01 01:01:01'),(355,20250219090511,1,'2020-01-01 01:01:01'),(356,20250219100000,1,'2020-01-01 01:01:01'),(357,20250219142401,1,'2020-01-01 01:01:01'),(358,20250224184002,1,'2020-01-01 01:01:01'),(359,20250225085436,1,'2020-01-01 01:01:01'),(360,20250226000000,1,'2020-01-01 01:01:01'),(361,20250226153445,1,'2020-01-01 01:01:01'),(362,20250304162702,1,'2020-01-01 01:01:01'),(363,20250306144233,1,'2020-01-01 01:01:01'),(364,20250313163430,1,'2020-01-01 01:01:01'),(365,20250317130944,1,'2020-01-01 01:01:01'),(366,20250318165922,1,'2020-01-01 01:01:01'),(367,20250320132525,1,'2020-01-01 01:01:01'),(368,20250320200000,1,'2020-01-01 01:01:01'),(369,20250326161930,1,'2020-01-01 01:01:01'),(370,20250326161931,1,'2020-01-01 01:01:01'),(371,20250331042354,1,'2020-01-01 01:01:01'),(372,20250331154206,1,'2020-01-01 01:01:01'),(373,20250401155831,1,'2020-01-01 01:01:01'),(374,20250408133233,1,'2020-01-01 01:01:01'),(375,20250410104321,1,'2020-01-01 01:01:01'),(376,20250421085116,1,'2020-01-01 01:01:01'),(377,20250422095806,1,'2020-01-01 01:01:01'),(378,20250424153059,1,'2020-01-01 01:01:01'),(379,20250430103833,1,'2020-01-01 01:01:01'),(380,20250430112622,1,'2020-01-01 01:01:01'),(381,20250501162727,1,'2020-01-01 01:01:01'),(382,20250502154517,1,'2020-01-01 01:01:01'),(383,20250502222222,1,'2020-01-01 01:01:01'),(384,20250507170845,1,'2020-01-01 01:01:01'),(385,20250513162912,1,'2020-01-01 01:01:01'),(386,20250519161614,1,'2020-01-01 01:01:01'),(387,20250519170000,1,'2020-01-01 01:01:01'),(388,20250520153848,1,'2020-01-01 01:01:01'),(389,20250528115932,1,'2020-01-01 01:01:01'),(390,20250529102706,1,'2020-01-01 01:01:01'),(391,20250603105558,1,'2020-01-01 01:01:01'),(392,20250609102714,1,'2020-01-01 01:01:01'),(393,20250609112613,1,'2020-01-01 01:01:01'),(394,20250613103810,1,'2020-01-01 01:01:01'),(395,20250616193950,1,'2020-01-01 01:01:01'),(396,20250624140757,1,'2020-01-01 01:01:01'),(397,20250626130239,1,'2020-01-01 01:01:01'),(398,20250629131032,1,'2020-01-01 01:01:01'),(399,20250701155654,1,'2020-01-01 01:01:01'),(400,20250707095725,1,'2020-01-01 01:01:01'),(401,20250716152435,1,'2020-01-01 01:01:01'),(402,20250718091828,1,'2020-01-01 01:01:01'),(403,20250728122229,1,'2020-01-01 01:01:01'),(404,20250731122715,1,'2020-01-01 01:01:01'),(405,20250731151000,1,'2020-01-01 01:01:01'),(406,20250803000000,1,'2020-01-01 01:01:01'),(407,20250805083116,1,'2020-01-01 01:01:01'),(408,20250807140441,1,'2020-01-01 01:01:01'),(409,20250808000000,1,'2020-01-01 01:01:01'),(410,20250811155036,1,'2020-01-01 01:01:01'),(411,20250813205039,1,'2020-01-01 01:01:01'),(412,20250814123333,1,'2020-01-01 01:01:01'),(413,20250815130115,1,'2020-01-01 01:01:01'),(414,20250816115553,1,'2020-01-01 01:01:01'),(415,20250817154557,1,'2020-01-01 01:01:01'),(416,20250825113751,1,'2020-01-01 01:01:01'),(417,20250827113140,1,'2020-01-01 01:01:01'),(418,20250828120836,1,'2020-01-01 01:01:01'),(419,20250902112642,1,'2020-01-01 01:01:01'),(420,20250904091745,1,'2020-01-01 01:01:01'),(421,20250905090000,1,'2020-01-01 01:01:01'),(422,20250922083056,1,'2020-01-01 01:01:01'),(423,20250923120000,1,'2020-01-01 01:01:01'),(424,20250926123048,1,'2020-01-01 01:01:01'),(425,20251015103505,1,'2020-01-01 01:01:01'),(426,20251015103600,1,'2020-01-01 01:01:01'),(427,20251015103700,1,'2020-01-01 01:01:01'),(428,20251015103800,1,'2020-01-01 01:01:01'),(429,20251015103900,1,'2020-01-01 01:01:01'),(430,20251028140000,1,'2020-01-01 01:01:01'),(431,20251028140100,1,'2020-01-01 01:01:01'),(432,20251028140110,1,'2020-01-01 01:01:01'),(433,20251028140200,1,'2020-01-01 01:01:01'),(434,20251028140300,1,'2020-01-01 01:01:01'),(435,20251028140400,1,'2020-01-01 01:01:01'),(436,20251031154558,1,'2020-01-01 01:01:01'),(437,20251103160848,1,'2020-01-01 01:01:01'),(438,20251104112849,1,'2020-01-01 01:01:01'),(439,20251106000000,1,'2020-01-01 01:01:01'),(440,20251107164629,1,'2020-01-01 01:01:01'),(441,20251107170854,1,'2020-01-01 01:01:01'),(442,20251110172137,1,'2020-01-01 01:01:01'),(443,20251111153133,1,'2020-01-01 01:01:01'),(444,20251117020000,1,'2020-01-01 01:01:01'),(445,20251117020100,1,'2020-01-01 01:01:01'),(446,20251117020200,1,'2020-01-01 01:01:01'),(447,20251121100000,1,'2020-01-01 01:01:01'),(448,20251121124239,1,'2020-01-01 01:01:01'),(449,20251124090450,1,'2020-01-01 01:01:01'),(450,20251124135808,1,'2020-01-01 01:01:01'),(451,20251124140138,1,'2020-01-01 01:01:01'),(452,20251124162948,1,'2020-01-01 01:01:01'),(453,20251127113559,1,'2020-01-01 01:01:01'),(454,20251202162232,1,'2020-01-01 01:01:01'),(455,20251203170808,1,'2020-01-01 01:01:01'),(456,20251207050413,1,'2020-01-01 01:01:01'),(457,20251208215800,1,'2020-01-01 01:01:01'),(458,20251209221730,1,'2020-01-01 01:01:01'),(459,20251209221850,1,'2020-01-01 01:01:01'),(460,20251215163721,1,'2020-01-01 01:01:01'),(461,20251217000000,1,'2020-01-01 01:01:01'),(462,20251217120000,1,'2020-01-01 01:01:01'),(463,20251229000000,1,'2020-01-01 01:01:01'),(464,20251229000010,1,'2020-01-01 01:01:01'),(465,20251229000020,1,'2020-01-01 01:01:01'),(466,20260106000000,1,'2020-01-01 01:01:01'),(467,20260108200708,1,'2020-01-01 01:01:01'),(468,20260108214732,1,'2020-01-01 01:01:01'),(469,20260109231821,1,'2020-01-01 01:01:01'),(470,20260113012054,1,'2020-01-01 01:01:01'),(471,20260124200020,1,'2020-01-01 01:01:01'),(472,20260126150840,1,'2020-01-01 01:01:01'),(473,20260126210724,1,'2020-01-01 01:01:01'),(474,20260202151756,1,'2020-01-01 01:01:01'),(475,20260205184907,1,'2020-01-01 01:01:01'),(476,20260210151544,1,'2020-01-01 01:01:01'),(477,20260210155109,1,'2020-01-01 01:01:01'),(478,20260210181120,1,'2020-01-01 01:01:01'),(479,20260211200153,1,'2020-01-01 01:01:01'),(480,20260217141240,1,'2020-01-01 01:01:01'),(481,20260217200906,1,'2020-01-01 01:01:01'),(482,20260218175704,1,'2020-01-01 01:01:01'),(483,20260314120000,1,'2020-01-01 01:01:01'),(484,20260316120000,1,'2020-01-01 01:01:01'),(485,20260316120001,1,'2020-01-01 01:01:01'),(486,20260316120002,1,'2020-01-01 01:01:01'),(487,20260316120003,1,'2020-01-01 01:01:01'),(488,20260316120004,1,'2020-01-01 01:01:01'),(489,20260316120005,1,'2020-01-01 01:01:01'),(490,20260316120006,1,'2020-01-01 01:01:01'),(491,20260316120007,1,'2020-01-01 01:01:01'),(492,20260316120008,1,'2020-01-01 01:01:01'),(493,20260316120009,1,'2020-01-01 01:01:01'),(494,20260316120010,1,'2020-01-01 01:01:01'),(495,20260317120000,1,'2020-01-01 01:01:01'),(496,20260318184559,1,'2020-01-01 01:01:01'),(497,20260319120000,1,'2020-01-01 01:01:01'),(498,20260323144117,1,'2020-01-01 01:01:01'),(499,20260324161944,1,'2020-01-01 01:01:01'),(500,20260324223334,1,'2020-01-01 01:01:01'),(501,20260326131501,1,'2020-01-01 01:01:01'),(502,20260326210603,1,'2020-01-01 01:01:01'),(503,20260331000000,1,'2020-01-01 01:01:01'),(504,20260401153000,1,'2020-01-01 01:01:01'),(505,20260401153001,1,'2020-01-01 01:01:01'),(506,20260401153503,1,'2020-01-01 01:01:01'),(507,20260403120000,1,'2020-01-01 01:01:01'),(508,20260409153713,1,'2020-01-01 01:01:01'),(509,20260409153714,1,'2020-01-01 01:01:01'),(510,20260409153715,1,'2020-01-01 01:01:01'),(511,20260409153716,1,'2020-01-01 01:01:01'),(512,20260409153717,1,'2020-01-01 01:01:01'),(513,20260409183610,1,'2020-01-01 01:01:01'),(514,20260410173222,1,'2020-01-01 01:01:01'),(515,20260422181702,1,'2020-01-01 01:01:01'),(516,20260423161823,1,'2020-01-01 01:01:01'),(517,20260423161824,1,'2020-01-01 01:01:01'),(518,20260518194422,1,'2020-01-01 01:01:01'),(519,20260522195224,1,'2020-01-01 01:01:01'),(520,20260522195225,1,'2020-01-01 01:01:01'),(521,20260522195226,1,'2020-01-01 01:01:01'),(522,20260522195227,1,'2020-01-01 01:01:01'),(523,20260522195229,1,'2020-01-01 01:01:01'),(524,20260522195230,1,'2020-01-01 01:01:01'),(525,20260522195231,1,'2020-01-01 01:01:01'),(526,20260522195232,1,'2020-01-01 01:01:01'),(527,20260522195233,1,'2020-01-01 01:01:01'),(528,20260522195234,1,'2020-01-01 01:01:01'),(529,20260522195235,1,'2020-01-01 01:01:01'),(530,20260527215817,1,'2020-01-01 01:01:01'),(531,20260528201143,1,'2020-01-01 01:01:01'),(532,20260528201150,1,'2020-01-01 01:01:01'),(533,20260528211626,1,'2020-01-01 01:01:01'),(534,20260528213326,1,'2020-01-01 01:01:01'),(535,20260529091823,1,'2020-01-01 01:01:01'),(536,20260529120000,1,'2020-01-01 01:01:01'),(537,20260601200727,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `mobile_device_management_solutions` ( @@ -3056,6 +3056,21 @@ CREATE TABLE `teams` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `trace_sampler_settings` ( + `id` tinyint unsigned NOT NULL, + `high_volume_ratio` double NOT NULL DEFAULT '0.001', + `standard_ratio` double NOT NULL DEFAULT '0.02', + `force_full` tinyint(1) NOT NULL DEFAULT '0', + `updated_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + CONSTRAINT `ck_trace_sampler_settings_high_range` CHECK ((`high_volume_ratio` between 0 and 1)), + CONSTRAINT `ck_trace_sampler_settings_singleton` CHECK ((`id` = 1)), + CONSTRAINT `ck_trace_sampler_settings_std_range` CHECK ((`standard_ratio` between 0 and 1)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `trace_sampler_settings` VALUES (1,0.001,0.02,0,'2026-06-01 00:00:00'); +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `upcoming_activities` ( `id` bigint unsigned NOT NULL AUTO_INCREMENT, `host_id` int unsigned NOT NULL, diff --git a/server/datastore/mysql/trace_sampler.go b/server/datastore/mysql/trace_sampler.go new file mode 100644 index 0000000000..2f765099c4 --- /dev/null +++ b/server/datastore/mysql/trace_sampler.go @@ -0,0 +1,56 @@ +package mysql + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/platform/tracing" + "github.com/jmoiron/sqlx" +) + +const traceSamplerSettingsID = 1 + +func (ds *Datastore) GetTraceSamplerSettings(ctx context.Context) (*tracing.Settings, error) { + const stmt = ` + SELECT high_volume_ratio, standard_ratio, force_full, updated_at + FROM trace_sampler_settings + WHERE id = ? + ` + var settings tracing.Settings + if err := sqlx.GetContext(ctx, ds.reader(ctx), &settings, stmt, traceSamplerSettingsID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ctxerr.Wrap(ctx, notFound("TraceSamplerSettings")) + } + return nil, ctxerr.Wrap(ctx, err, "get trace_sampler_settings") + } + return &settings, nil +} + +func (ds *Datastore) SetTraceSamplerSettings(ctx context.Context, settings *tracing.Settings) error { + const stmt = ` + UPDATE trace_sampler_settings + SET high_volume_ratio = ?, standard_ratio = ?, force_full = ? + WHERE id = ? + ` + res, err := ds.writer(ctx).ExecContext(ctx, stmt, + settings.HighVolumeRatio, + settings.StandardRatio, + settings.ForceFull, + traceSamplerSettingsID, + ) + if err != nil { + return ctxerr.Wrap(ctx, err, "set trace_sampler_settings") + } + rows, err := res.RowsAffected() + if err != nil { + return ctxerr.Wrap(ctx, err, "set trace_sampler_settings: rows affected") + } + // The singleton row is seeded by the migration. A missing row means the invariant is broken. + if rows != 1 { + return ctxerr.Wrap(ctx, fmt.Errorf("set trace_sampler_settings: expected 1 row updated, got %d", rows)) + } + return nil +} diff --git a/server/datastore/mysql/trace_sampler_test.go b/server/datastore/mysql/trace_sampler_test.go new file mode 100644 index 0000000000..137dab43ad --- /dev/null +++ b/server/datastore/mysql/trace_sampler_test.go @@ -0,0 +1,65 @@ +package mysql + +import ( + "testing" + + "github.com/fleetdm/fleet/v4/server/platform/tracing" + "github.com/stretchr/testify/require" +) + +func TestTraceSamplerSettings(t *testing.T) { + ds := CreateMySQLDS(t) + ctx := t.Context() + + t.Run("seeded defaults are returned", func(t *testing.T) { + got, err := ds.GetTraceSamplerSettings(ctx) + require.NoError(t, err) + require.NotNil(t, got) + require.InDelta(t, 0.001, got.HighVolumeRatio, 1e-9) + require.InDelta(t, 0.02, got.StandardRatio, 1e-9) + require.False(t, got.ForceFull) + }) + + t.Run("round trip persists changes", func(t *testing.T) { + err := ds.SetTraceSamplerSettings(ctx, &tracing.Settings{ + HighVolumeRatio: 0.005, + StandardRatio: 0.1, + ForceFull: true, + }) + require.NoError(t, err) + + got, err := ds.GetTraceSamplerSettings(ctx) + require.NoError(t, err) + require.InDelta(t, 0.005, got.HighVolumeRatio, 1e-9) + require.InDelta(t, 0.1, got.StandardRatio, 1e-9) + require.True(t, got.ForceFull) + }) + + t.Run("out of range ratio rejected by CHECK constraint", func(t *testing.T) { + err := ds.SetTraceSamplerSettings(ctx, &tracing.Settings{ + HighVolumeRatio: 1.5, + StandardRatio: 0.02, + ForceFull: false, + }) + require.Error(t, err) + }) + + t.Run("set fails when singleton row is missing", func(t *testing.T) { + // Locks in the RowsAffected != 1 guard in SetTraceSamplerSettings. If the seeded singleton row is missing (DB invariant + // broken), Set must surface a loud error rather than silently no-op. + _, err := ds.writer(ctx).ExecContext(ctx, `DELETE FROM trace_sampler_settings`) + require.NoError(t, err) + t.Cleanup(func() { + _, err := ds.writer(ctx).ExecContext(ctx, `INSERT INTO trace_sampler_settings (id) VALUES (1)`) + require.NoError(t, err) + }) + + err = ds.SetTraceSamplerSettings(ctx, &tracing.Settings{ + HighVolumeRatio: 0.01, + StandardRatio: 0.05, + ForceFull: false, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "expected 1 row updated") + }) +} diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index f98d762c8a..b4a444e845 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -20,6 +20,7 @@ import ( "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/storage" platform_errors "github.com/fleetdm/fleet/v4/server/platform/errors" platform_http "github.com/fleetdm/fleet/v4/server/platform/http" + "github.com/fleetdm/fleet/v4/server/platform/tracing" "github.com/jmoiron/sqlx" ) @@ -3344,6 +3345,16 @@ type Datastore interface { // HasWindowsUpdateConfigProfileConfigured checks if a profile for the team already exists in the update_settings table. HasWindowsUpdateConfigProfileConfigured(ctx context.Context, teamID uint) (bool, error) + + /////////////////////////////////////////////////////////////////////////////// + // TraceSamplerStore + + // GetTraceSamplerSettings returns the singleton trace_sampler_settings row. + GetTraceSamplerSettings(ctx context.Context) (*tracing.Settings, error) + + // SetTraceSamplerSettings updates the singleton trace_sampler_settings row. The caller is responsible for validating that + // ratios are in [0, 1]. The DB CHECK constraints reject out of range writes as a backstop. + SetTraceSamplerSettings(ctx context.Context, settings *tracing.Settings) error } type AndroidDatastore interface { diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index bb4e133f71..fe438600d8 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -18,6 +18,7 @@ import ( "github.com/fleetdm/fleet/v4/server/mdm/apple/mobileconfig" "github.com/fleetdm/fleet/v4/server/mdm/nanodep/godep" "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/mdm" + "github.com/fleetdm/fleet/v4/server/platform/tracing" "github.com/jmoiron/sqlx" ) @@ -2047,6 +2048,10 @@ type HasAppleUpdateConfigProfileConfiguredFunc func(ctx context.Context, teamID type HasWindowsUpdateConfigProfileConfiguredFunc func(ctx context.Context, teamID uint) (bool, error) +type GetTraceSamplerSettingsFunc func(ctx context.Context) (*tracing.Settings, error) + +type SetTraceSamplerSettingsFunc func(ctx context.Context, settings *tracing.Settings) error + type DataStore struct { AppConfigFunc AppConfigFunc AppConfigFuncInvoked bool @@ -5084,6 +5089,12 @@ type DataStore struct { HasWindowsUpdateConfigProfileConfiguredFunc HasWindowsUpdateConfigProfileConfiguredFunc HasWindowsUpdateConfigProfileConfiguredFuncInvoked bool + GetTraceSamplerSettingsFunc GetTraceSamplerSettingsFunc + GetTraceSamplerSettingsFuncInvoked bool + + SetTraceSamplerSettingsFunc SetTraceSamplerSettingsFunc + SetTraceSamplerSettingsFuncInvoked bool + mu sync.Mutex } @@ -12170,3 +12181,17 @@ func (s *DataStore) HasWindowsUpdateConfigProfileConfigured(ctx context.Context, s.mu.Unlock() return s.HasWindowsUpdateConfigProfileConfiguredFunc(ctx, teamID) } + +func (s *DataStore) GetTraceSamplerSettings(ctx context.Context) (*tracing.Settings, error) { + s.mu.Lock() + s.GetTraceSamplerSettingsFuncInvoked = true + s.mu.Unlock() + return s.GetTraceSamplerSettingsFunc(ctx) +} + +func (s *DataStore) SetTraceSamplerSettings(ctx context.Context, settings *tracing.Settings) error { + s.mu.Lock() + s.SetTraceSamplerSettingsFuncInvoked = true + s.mu.Unlock() + return s.SetTraceSamplerSettingsFunc(ctx, settings) +} diff --git a/server/platform/tracing/doc.go b/server/platform/tracing/doc.go new file mode 100644 index 0000000000..21f0e42eaf --- /dev/null +++ b/server/platform/tracing/doc.go @@ -0,0 +1,13 @@ +// Package tracing implements a route aware head sampler for Fleet's OTEL trace export. The sampler classifies each span via a +// Registry. Each bounded context populates the Registry at startup with its own routes. The sampler then applies per tier ratio +// sampling. The goal is to prevent noisy hot agent paths from drowning out the rare load bearing paths (enroll, MDM command +// flows, cron jobs). +// +// Runtime control lives in the trace_sampler_settings MySQL row. Each Fleet replica runs StartSettingsPoller which re-reads the +// row every 60 seconds and atomically swaps the sampler's state. No restart is required to flip force_full during an incident +// debug window. +// +// Architecture: platform/tracing owns the mechanism (sampler, tier enum, registry). Each bounded context owns the policy: which +// of its routes belong in which tier. Each context registers them at startup. This keeps the platform package free of cross +// context coupling. +package tracing diff --git a/server/platform/tracing/registry.go b/server/platform/tracing/registry.go new file mode 100644 index 0000000000..c235287854 --- /dev/null +++ b/server/platform/tracing/registry.go @@ -0,0 +1,92 @@ +package tracing + +import ( + "regexp" + "sync" +) + +// Tier classifies a span for sampling. Higher value spans (TierAlways) always get sampled. High volume noise (TierHighVolume) +// is downsampled aggressively. Liveness probes (TierNever) are dropped unconditionally. +type Tier int + +const ( + // TierAlways is the catch all. Spans not registered with the Registry fall here, and are sampled at 100%. Cron jobs, enroll, + // MDM checkin, SCEP, GitOps batch (all the load bearing flows) should never need an explicit registration. Their absence from + // the noisy lists is what keeps them safe by default. + TierAlways Tier = iota + + // TierHighVolume is for routes that dominate request volume without being individually interesting (osquery distributed + // read/write, orbit ping/config, device desktop/ping). Sampled at the configured high volume ratio (default 0.1%). + TierHighVolume + + // TierStandard is for routes with moderate volume and moderate diagnostic value (admin reads, dashboard endpoints, asset + // paths). Sampled at the configured standard ratio (default 2%). + TierStandard + + // TierNever drops the span unconditionally, even under ForceFull. Reserved for high volume zero diagnostic value paths like + // liveness probes, the version endpoint, and the Prometheus scrape path. + TierNever +) + +// Registry maps normalized route names to tiers. Bounded contexts register their own routes at startup via Register. The +// sampler reads via Lookup on every span. Routes not present in the registry fall to TierAlways. +// +// The Registry is the policy seam. platform/tracing provides the mechanism (sampler and tier enum). Each bounded context +// provides the policy: which of its routes belong in which tier. +type Registry struct { + mu sync.RWMutex + routes map[string]Tier +} + +// NewRegistry returns an empty Registry. Routes are added via Register. +func NewRegistry() *Registry { + return &Registry{routes: make(map[string]Tier)} +} + +// Register classifies a method and path. Paths are normalized on write the same way they are on Lookup. The gorilla/mux +// version segment is collapsed to "_version_". Regex constrained params (e.g. "{id:[0-9]+}") are stripped to "{id}". Callers +// can therefore register in either the readable form ("/api/_version_/fleet/hosts/{id}") or the raw template form, and lookups +// will still match. Both alternate path forms (e.g. "/api/v1/osquery/..." and "/api/osquery/...") still need separate +// registrations. +// +// Re-registering the same normalized method+path overwrites the prior tier. +func (r *Registry) Register(method, path string, tier Tier) { + r.mu.Lock() + defer r.mu.Unlock() + r.routes[normalizeSpanName(spanKey(method, path))] = tier +} + +// Lookup returns the tier for a span name and a bool indicating whether the route was found in the registry. Span names are +// normalized before lookup so the gorilla version regex is stripped to "_version_". Cron and other non HTTP span names (no +// leading "METHOD ") simply won't be in the registry and return (TierAlways, false). +func (r *Registry) Lookup(spanName string) (Tier, bool) { + normalized := normalizeSpanName(spanName) + r.mu.RLock() + defer r.mu.RUnlock() + t, ok := r.routes[normalized] + return t, ok +} + +// versionTemplatePattern matches the gorilla/mux version segment Fleet inserts via /_version_/, replacing it with +// /{fleetversion:(?:v1|2022-04|latest)}/ at registration time. We normalize it back to /_version_/ so the registry stays +// decoupled from the configured version list. +var versionTemplatePattern = regexp.MustCompile(`\{fleetversion:[^}]*\}`) + +// muxParamRegexPattern strips regex constraints from gorilla/mux path params. For example {id:[0-9]+} becomes {id}, and +// {fleet_id:[0-9]+} becomes {fleet_id}. Fleet uses this style extensively, and the route tier policy in +// server/service/tracing_tiers.go registers the simpler {id} form. Without this normalization, spans whose mux template +// includes the regex would miss the registry and silently fall to TierAlways at 100% sampling. +var muxParamRegexPattern = regexp.MustCompile(`\{([a-zA-Z0-9_]+):[^}]+\}`) + +func normalizeSpanName(name string) string { + // Apply versionTemplatePattern first because it strips both the param name and the surrounding braces (replacing with + // `_version_`). muxParamRegexPattern preserves the braces. Running it first would turn {fleetversion:...} into + // {fleetversion}, which would then no longer match versionTemplatePattern. + name = versionTemplatePattern.ReplaceAllString(name, "_version_") + name = muxParamRegexPattern.ReplaceAllString(name, "{$1}") + return name +} + +func spanKey(method, path string) string { + return method + " " + path +} diff --git a/server/platform/tracing/registry_test.go b/server/platform/tracing/registry_test.go new file mode 100644 index 0000000000..f84b64541d --- /dev/null +++ b/server/platform/tracing/registry_test.go @@ -0,0 +1,176 @@ +package tracing + +import ( + "net/http" + "strconv" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRegistry_LookupNormalizes(t *testing.T) { + // One fixture covers all input shapes that should resolve to the same registered route: fleetversion templates (multi + // version and single version), mux regex constrained params, and already normalized inputs. + r := NewRegistry() + r.Register(http.MethodGet, "/api/_version_/fleet/hosts", TierStandard) + r.Register(http.MethodGet, "/api/_version_/fleet/hosts/{id}", TierStandard) + r.Register(http.MethodPatch, "/api/_version_/fleet/fleets/{fleet_id}/secrets", TierAlways) + + cases := []struct { + name string + input string + want Tier + wantOK bool + }{ + { + name: "regex version template", + input: "GET /api/{fleetversion:(?:v1|2022-04|latest)}/fleet/hosts", + want: TierStandard, + wantOK: true, + }, + { + name: "single version regex template", + input: "GET /api/{fleetversion:(?:latest)}/fleet/hosts", + want: TierStandard, + wantOK: true, + }, + { + name: "already normalized form", + input: "GET /api/_version_/fleet/hosts", + want: TierStandard, + wantOK: true, + }, + { + name: "mux regex param {id:[0-9]+}", + input: "GET /api/{fleetversion:(?:v1|2022-04|latest)}/fleet/hosts/{id:[0-9]+}", + want: TierStandard, + wantOK: true, + }, + { + name: "mux regex param {fleet_id:[0-9]+}", + input: "PATCH /api/{fleetversion:(?:v1|2022-04|latest)}/fleet/fleets/{fleet_id:[0-9]+}/secrets", + want: TierAlways, + wantOK: true, + }, + { + name: "unregistered route", + input: "POST /not/in/registry", + want: TierAlways, // zero value. Sampler interprets the !ok as the catch all. + wantOK: false, + }, + { + name: "cron span name is not in registry", + input: "vuln.update_host_counts", + want: TierAlways, + wantOK: false, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, ok := r.Lookup(c.input) + require.Equal(t, c.wantOK, ok) + require.Equal(t, c.want, got) + }) + } +} + +func TestRegistry_RegisterNormalizesSymmetrically(t *testing.T) { + // Inverse of TestRegistry_LookupNormalizes: registering with the regex form must also resolve via lookup of the bare form. + // This is the invariant the Register side normalization delivers. + r := NewRegistry() + r.Register(http.MethodGet, "/api/_version_/fleet/hosts/{id:[0-9]+}", TierStandard) + + got, ok := r.Lookup("GET /api/{fleetversion:(?:latest)}/fleet/hosts/{id}") + require.True(t, ok, "regex form registration must be findable via simple form lookup") + require.Equal(t, TierStandard, got) +} + +func TestRegistry_RegisterOverwrites(t *testing.T) { + t.Run("same form", func(t *testing.T) { + r := NewRegistry() + r.Register(http.MethodPost, "/foo", TierStandard) + r.Register(http.MethodPost, "/foo", TierHighVolume) + + got, ok := r.Lookup("POST /foo") + require.True(t, ok) + require.Equal(t, TierHighVolume, got) + }) + + t.Run("different forms collide on the same normalized key", func(t *testing.T) { + // {id} and {id:[0-9]+} are the same logical route after normalization. The second Register must overwrite the first + // regardless of which surface form was used. + r := NewRegistry() + r.Register(http.MethodGet, "/api/_version_/fleet/hosts/{id}", TierStandard) + r.Register(http.MethodGet, "/api/_version_/fleet/hosts/{id:[0-9]+}", TierHighVolume) + + got, _ := r.Lookup("GET /api/_version_/fleet/hosts/{id}") + require.Equal(t, TierHighVolume, got) + }) +} + +func TestRegistry_ConcurrentReadersAndWriters(t *testing.T) { + // Exercise the RWMutex under -race. Late arriving registrations must not corrupt concurrent lookups. This is what makes it + // safe for bounded contexts to register at startup while the tracer provider is already serving spans. + r := NewRegistry() + const writerCount = 4 + const readerCount = 8 + const iterations = 5000 + + var wg sync.WaitGroup + for w := range writerCount { + wg.Go(func() { + for i := range iterations { + path := "/path/writer/" + strconv.Itoa(w) + "/" + strconv.Itoa(i) + r.Register(http.MethodGet, path, TierStandard) + } + }) + } + for range readerCount { + wg.Go(func() { + for range iterations { + _, _ = r.Lookup("GET /path/writer/0/0") + } + }) + } + wg.Wait() +} + +func TestNormalizeSpanName(t *testing.T) { + // Unit test for the helper. Lookup integration is covered separately. These cases pin the helper's behavior independent of + // the map lookup. + cases := []struct { + in, want string + }{ + {"GET /healthz", "GET /healthz"}, + { + "GET /api/{fleetversion:(?:v1|2022-04|latest)}/fleet/hosts", + "GET /api/_version_/fleet/hosts", + }, + { + "GET /api/_version_/fleet/queries", + "GET /api/_version_/fleet/queries", + }, + {"vuln.update_host_counts", "vuln.update_host_counts"}, + // Mux regex constraints on path params are stripped to the bare {name} form so the registry can stay decoupled from the + // constraint syntax. + { + "GET /api/_version_/fleet/hosts/{id:[0-9]+}", + "GET /api/_version_/fleet/hosts/{id}", + }, + { + "PATCH /api/{fleetversion:(?:v1|2022-04|latest)}/fleet/fleets/{fleet_id:[0-9]+}/secrets", + "PATCH /api/_version_/fleet/fleets/{fleet_id}/secrets", + }, + // Params without a regex constraint pass through unchanged. + { + "GET /api/_version_/fleet/device/{token}/desktop", + "GET /api/_version_/fleet/device/{token}/desktop", + }, + } + for _, c := range cases { + t.Run(c.in, func(t *testing.T) { + require.Equal(t, c.want, normalizeSpanName(c.in)) + }) + } +} diff --git a/server/platform/tracing/sampler.go b/server/platform/tracing/sampler.go new file mode 100644 index 0000000000..b6c6f92fd2 --- /dev/null +++ b/server/platform/tracing/sampler.go @@ -0,0 +1,105 @@ +package tracing + +import ( + "fmt" + "sync/atomic" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +// Default sampling ratios. These match the seeded trace_sampler_settings row so a freshly started server uses the same ratios +// as one that has polled the DB. +const ( + DefaultHighVolumeRatio = 0.001 + DefaultStandardRatio = 0.02 +) + +// RouteTierSampler implements sdktrace.Sampler. The configured ratios live in an atomic.Pointer so SettingsPoller can swap them +// under a hot reader without locking. Tier classification is delegated to the Registry passed at construction time. +type RouteTierSampler struct { + state atomic.Pointer[samplerState] + registry *Registry +} + +type samplerState struct { + highVolume sdktrace.Sampler + standard sdktrace.Sampler + always sdktrace.Sampler + never sdktrace.Sampler + forceFull bool +} + +// NewRouteTierSampler returns a sampler initialized with the default ratios and force_full=false. The poller (if running) +// overwrites these on the first tick once it reads the DB row. The registry is consulted on every ShouldSample call. Routes +// added to the registry after construction are picked up immediately. +func NewRouteTierSampler(registry *Registry) *RouteTierSampler { + s := &RouteTierSampler{registry: registry} + s.state.Store(buildState(DefaultHighVolumeRatio, DefaultStandardRatio, false)) + return s +} + +// Apply replaces the sampler's state atomically. Out of range ratios are clamped to [0, 1] as a defensive backstop. The DB +// CHECK constraints and the PATCH handler validation reject these earlier in the pipeline. +func (s *RouteTierSampler) Apply(highVolume, standard float64, forceFull bool) { + s.state.Store(buildState(clamp01(highVolume), clamp01(standard), forceFull)) +} + +func clamp01(v float64) float64 { + switch { + case v < 0: + return 0 + case v > 1: + return 1 + } + return v +} + +func buildState(highVolume, standard float64, forceFull bool) *samplerState { + return &samplerState{ + highVolume: sdktrace.TraceIDRatioBased(highVolume), + standard: sdktrace.TraceIDRatioBased(standard), + always: sdktrace.AlwaysSample(), + never: sdktrace.NeverSample(), + forceFull: forceFull, + } +} + +// ShouldSample implements sdktrace.Sampler. TierNever wins over ForceFull. Liveness probes should never trace, even during a +// 100% debug window. Unregistered spans (cron, MDM checkin, enroll, etc.) fall to TierAlways. +func (s *RouteTierSampler) ShouldSample(p sdktrace.SamplingParameters) sdktrace.SamplingResult { + st := s.state.Load() + tier, _ := s.registry.Lookup(p.Name) + if tier == TierNever { + return st.never.ShouldSample(p) + } + if st.forceFull { + return st.always.ShouldSample(p) + } + switch tier { + case TierHighVolume: + return st.highVolume.ShouldSample(p) + case TierStandard: + return st.standard.ShouldSample(p) + case TierAlways: + return st.always.ShouldSample(p) + } + return st.always.ShouldSample(p) +} + +// Description implements sdktrace.Sampler. OTel uses it for diagnostic logging. The value should describe the sampler's +// behavior unambiguously. +func (s *RouteTierSampler) Description() string { + st := s.state.Load() + return fmt.Sprintf("RouteTierSampler{highVolume=%g,standard=%g,forceFull=%t}", + samplerRatio(st.highVolume), samplerRatio(st.standard), st.forceFull) +} + +// samplerRatio extracts the configured ratio from a TraceIDRatioBased sampler for description purposes only. The SDK does not +// expose the ratio directly, so we parse its description ("TraceIDRatioBased{0.001}"). +func samplerRatio(s sdktrace.Sampler) float64 { + var r float64 + if _, err := fmt.Sscanf(s.Description(), "TraceIDRatioBased{%f}", &r); err != nil { + return -1 + } + return r +} diff --git a/server/platform/tracing/sampler_test.go b/server/platform/tracing/sampler_test.go new file mode 100644 index 0000000000..fff05b121c --- /dev/null +++ b/server/platform/tracing/sampler_test.go @@ -0,0 +1,200 @@ +package tracing + +import ( + "encoding/binary" + "math/rand/v2" + "net/http" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" +) + +// testRegistry returns a Registry pre-populated with the routes the sampler tests exercise. It lives next to its only consumer +// so registry_test.go can stay focused on Registry semantics. +func testRegistry() *Registry { + r := NewRegistry() + r.Register(http.MethodGet, "/healthz", TierNever) + r.Register(http.MethodGet, "/version", TierNever) + r.Register(http.MethodGet, "/metrics", TierNever) + r.Register(http.MethodPost, "/api/osquery/distributed/read", TierHighVolume) + r.Register(http.MethodPost, "/api/v1/osquery/distributed/read", TierHighVolume) + r.Register(http.MethodPost, "/api/osquery/distributed/write", TierHighVolume) + r.Register(http.MethodPost, "/api/fleet/orbit/config", TierHighVolume) + r.Register(http.MethodHead, "/api/fleet/orbit/ping", TierHighVolume) + r.Register(http.MethodHead, "/api/_version_/fleet/device/{token}/ping", TierHighVolume) + r.Register(http.MethodGet, "/api/_version_/fleet/device/{token}/desktop", TierHighVolume) + r.Register(http.MethodGet, "/api/_version_/fleet/hosts", TierStandard) + r.Register(http.MethodGet, "/api/_version_/fleet/hosts/{id}", TierStandard) + r.Register(http.MethodGet, "/api/_version_/fleet/queries", TierStandard) + return r +} + +// sample is a tiny helper that asks the sampler whether a span with the given name should be recorded. A pseudo random trace +// ID is used so the TraceIDRatioBased sampler's decision varies per call. +// +//nolint:gosec // test trace IDs, not security sensitive +func sample(t *testing.T, s *RouteTierSampler, name string) bool { + t.Helper() + var tid trace.TraceID + binary.LittleEndian.PutUint64(tid[0:8], rand.Uint64()) + binary.LittleEndian.PutUint64(tid[8:16], rand.Uint64()) + res := s.ShouldSample(sdktrace.SamplingParameters{ + TraceID: tid, + Name: name, + Kind: trace.SpanKindServer, + }) + return res.Decision == sdktrace.RecordAndSample +} + +// sampleRate runs N trials and returns the observed sample rate. +func sampleRate(t *testing.T, s *RouteTierSampler, name string, n int) float64 { + t.Helper() + hits := 0 + for range n { + if sample(t, s, name) { + hits++ + } + } + return float64(hits) / float64(n) +} + +// TestRouteTierSampler_NeverTierDropsUnconditionally locks in the invariant that TierNever paths are never sampled, both at +// default config and under the most aggressive override (force_full=true with ratios maxed). The force_full subtest is the +// stronger guarantee. The default config case is kept to make the absence of any default-time leak explicit. +func TestRouteTierSampler_NeverTierDropsUnconditionally(t *testing.T) { + paths := []string{"GET /healthz", "GET /version", "GET /metrics"} + + cases := []struct { + name string + apply func(*RouteTierSampler) + }{ + {name: "default config", apply: func(*RouteTierSampler) {}}, + {name: "force_full with max ratios", apply: func(s *RouteTierSampler) { s.Apply(1.0, 1.0, true) }}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + s := NewRouteTierSampler(testRegistry()) + c.apply(s) + for _, p := range paths { + for range 1000 { + require.False(t, sample(t, s, p), "tierNever must drop %s", p) + } + } + }) + } +} + +// TestRouteTierSampler_AlwaysTierKeepsUnconditionally locks in the invariant that unclassified spans (cron, novel routes) +// always sample, both at default config and when ratios are forced to zero. The ratios=0 subtest is the stronger guarantee. +func TestRouteTierSampler_AlwaysTierKeepsUnconditionally(t *testing.T) { + names := []string{ + "vuln.update_host_counts", // cron + "POST /api/_version_/fleet/mdm/profiles/batch", // GitOps batch + "POST /api/fleet/orbit/enroll", // enroll + "some-future-endpoint-not-in-any-list", // unknown + } + + cases := []struct { + name string + apply func(*RouteTierSampler) + }{ + {name: "default config", apply: func(*RouteTierSampler) {}}, + {name: "ratios forced to zero", apply: func(s *RouteTierSampler) { s.Apply(0.0, 0.0, false) }}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + s := NewRouteTierSampler(testRegistry()) + c.apply(s) + for _, n := range names { + for range 1000 { + require.True(t, sample(t, s, n), "tierAlways must keep %s", n) + } + } + }) + } +} + +func TestRouteTierSampler_RatioSampling(t *testing.T) { + const n = 100_000 + const tolerance = 0.005 // 0.5pp absolute, generous for 100k trials + + s := NewRouteTierSampler(testRegistry()) + s.Apply(0.1, 0.5, false) // visible ratios for stable comparisons + + highRate := sampleRate(t, s, "POST /api/osquery/distributed/read", n) + require.InDelta(t, 0.1, highRate, tolerance, "high volume tier should track its configured ratio") + + stdRate := sampleRate(t, s, "GET /api/_version_/fleet/hosts", n) + require.InDelta(t, 0.5, stdRate, tolerance, "standard tier should track its configured ratio") +} + +func TestRouteTierSampler_ForceFull(t *testing.T) { + s := NewRouteTierSampler(testRegistry()) + s.Apply(0.0, 0.0, true) // ratios zero, force_full should override + + for _, name := range []string{ + "POST /api/osquery/distributed/read", // would be 0% via high volume + "GET /api/_version_/fleet/hosts", // would be 0% via standard + "POST /api/fleet/orbit/enroll", // already always + } { + t.Run(name, func(t *testing.T) { + for range 1000 { + require.True(t, sample(t, s, name), + "force_full must override ratio based tiers") + } + }) + } +} + +func TestRouteTierSampler_ApplyRaceFree(t *testing.T) { + s := NewRouteTierSampler(testRegistry()) + + var ( + stop atomic.Bool + readers sync.WaitGroup + writers sync.WaitGroup + ) + + // One writer flips ratios continuously. + writers.Go(func() { + for !stop.Load() { + //nolint:gosec // test fuzz inputs, not security sensitive + s.Apply(rand.Float64(), rand.Float64(), rand.IntN(2) == 0) + } + }) + + // Several readers hammer ShouldSample. + const readerCount = 8 + for range readerCount { + readers.Go(func() { + for !stop.Load() { + _ = sample(t, s, "POST /api/osquery/distributed/read") + _ = sample(t, s, "GET /healthz") + _ = sample(t, s, "vuln.update_host_counts") + } + }) + } + + // Run for a tight window. The race detector will fail if there's a torn read. + for range 50_000 { + _ = sample(t, s, "POST /api/_version_/fleet/spec/teams") + } + stop.Store(true) + readers.Wait() + writers.Wait() +} + +func TestRouteTierSampler_ClampOutOfRange(t *testing.T) { + // Apply should clamp defensively even if a caller passes out of range ratios. The DB CHECK rejects these in practice. + s := NewRouteTierSampler(testRegistry()) + s.Apply(-1.0, 5.0, false) + st := s.state.Load() + require.Equal(t, "TraceIDRatioBased{0}", st.highVolume.Description()) + require.Equal(t, "TraceIDRatioBased{1}", st.standard.Description()) +} diff --git a/server/platform/tracing/settings.go b/server/platform/tracing/settings.go new file mode 100644 index 0000000000..6991e240cd --- /dev/null +++ b/server/platform/tracing/settings.go @@ -0,0 +1,13 @@ +package tracing + +import "time" + +// Settings is the runtime tunable head sampling configuration. The settings live in the trace_sampler_settings singleton row +// and are polled by each Fleet replica so support can adjust sampling without a restart. +type Settings struct { + HighVolumeRatio float64 `json:"high_volume_ratio" db:"high_volume_ratio"` + StandardRatio float64 `json:"standard_ratio" db:"standard_ratio"` + ForceFull bool `json:"force_full" db:"force_full"` + // UpdatedAt uses omitzero so the PATCH handler can zero it before echoing the response. + UpdatedAt time.Time `json:"updated_at,omitzero" db:"updated_at"` +} diff --git a/server/platform/tracing/settings_poller.go b/server/platform/tracing/settings_poller.go new file mode 100644 index 0000000000..fae99bfed8 --- /dev/null +++ b/server/platform/tracing/settings_poller.go @@ -0,0 +1,72 @@ +package tracing + +import ( + "context" + "log/slog" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" +) + +// settingsPollInterval is how often each Fleet replica re-reads trace_sampler_settings. 60s matches industry defaults for +// feature flag style runtime config. +const settingsPollInterval = 60 * time.Second + +// pollTracer instruments the poll loop. When OTEL is not configured the global provider returns a no-op tracer, so this is +// free when tracing is disabled. +var pollTracer = otel.Tracer("github.com/fleetdm/fleet/v4/server/platform/tracing") + +// settingsReader is the minimal datastore surface the poller needs. The full fleet.Datastore is large. Depending only on this +// interface keeps the poller's tests cheap and the package free of cross context coupling. +type settingsReader interface { + GetTraceSamplerSettings(ctx context.Context) (*Settings, error) +} + +// StartSettingsPoller runs the polling loop until ctx is cancelled. On each tick it reads the trace_sampler_settings row, +// compares the values to the last applied state, and calls sampler. Apply only when something changed. +// +// On startup it does one immediate read so the sampler picks up the row's current values without waiting a full interval. If +// the first read fails, the sampler keeps its compile time defaults and a warning is logged. The next tick will try again. +func StartSettingsPoller(ctx context.Context, sampler *RouteTierSampler, ds settingsReader, logger *slog.Logger) { + pollAndApply := func(last *Settings) *Settings { + spanCtx, span := pollTracer.Start(ctx, "tracing.poll_settings", + trace.WithNewRoot(), + trace.WithSpanKind(trace.SpanKindInternal), + ) + defer span.End() + + got, err := ds.GetTraceSamplerSettings(spanCtx) + if err != nil { + logger.ErrorContext(spanCtx, "trace sampler settings poll failed", "err", err) + return last + } + if last != nil && + got.HighVolumeRatio == last.HighVolumeRatio && + got.StandardRatio == last.StandardRatio && + got.ForceFull == last.ForceFull { + return last + } + sampler.Apply(got.HighVolumeRatio, got.StandardRatio, got.ForceFull) + logger.InfoContext(spanCtx, "trace sampler settings applied", + "high_volume_ratio", got.HighVolumeRatio, + "standard_ratio", got.StandardRatio, + "force_full", got.ForceFull, + ) + return got + } + + var last *Settings + last = pollAndApply(last) + + ticker := time.NewTicker(settingsPollInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + last = pollAndApply(last) + } + } +} diff --git a/server/platform/tracing/settings_poller_test.go b/server/platform/tracing/settings_poller_test.go new file mode 100644 index 0000000000..1d4bd1e5c9 --- /dev/null +++ b/server/platform/tracing/settings_poller_test.go @@ -0,0 +1,139 @@ +package tracing + +import ( + "context" + "errors" + "io" + "log/slog" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/require" +) + +type stubReader struct { + mu atomic.Pointer[Settings] + err atomic.Pointer[error] + getCalls atomic.Int32 +} + +func (s *stubReader) set(settings Settings) { + s.mu.Store(&settings) +} + +func (s *stubReader) setErr(err error) { + s.err.Store(&err) +} + +func (s *stubReader) GetTraceSamplerSettings(_ context.Context) (*Settings, error) { + s.getCalls.Add(1) + if e := s.err.Load(); e != nil && *e != nil { + return nil, *e + } + if cur := s.mu.Load(); cur != nil { + out := *cur + return &out, nil + } + return &Settings{ + HighVolumeRatio: DefaultHighVolumeRatio, + StandardRatio: DefaultStandardRatio, + }, nil +} + +func discardLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +func TestStartSettingsPoller_AppliesInitialReadImmediately(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + r := &stubReader{} + r.set(Settings{ + HighVolumeRatio: 0.4, + StandardRatio: 0.8, + ForceFull: true, + }) + + sampler := NewRouteTierSampler(NewRegistry()) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + go StartSettingsPoller(ctx, sampler, r, discardLogger()) + + // Wait blocks until every other goroutine in the bubble is durably blocked. The poller does its synchronous initial + // read, applies, then blocks on the ticker. So once Wait returns, the apply has happened. + synctest.Wait() + + require.Equal(t, int32(1), r.getCalls.Load(), "exactly one poll should have happened by now") + st := sampler.state.Load() + require.True(t, st.forceFull, "initial read must apply force_full") + }) +} + +func TestStartSettingsPoller_HandlesErrorGracefully(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + r := &stubReader{} + r.setErr(errors.New("db unavailable")) + + sampler := NewRouteTierSampler(NewRegistry()) + // Capture current state. It should remain unchanged after a failed poll. + beforeForceFull := sampler.state.Load().forceFull + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + go StartSettingsPoller(ctx, sampler, r, discardLogger()) + + synctest.Wait() + + require.Equal(t, int32(1), r.getCalls.Load(), "the failed poll still counts as one read") + require.Equal(t, beforeForceFull, sampler.state.Load().forceFull, + "sampler state must be unchanged when the read fails") + }) +} + +func TestStartSettingsPoller_AppliesChangeOnTick(t *testing.T) { + // Locks in the actual 60s ticker behavior. The old tests could only assert the initial synchronous read because waiting a + // real minute per test was untenable. With synctest, advancing time is free. + synctest.Test(t, func(t *testing.T) { + r := &stubReader{} + r.set(Settings{ + HighVolumeRatio: 0.4, + StandardRatio: 0.8, + ForceFull: true, + }) + + sampler := NewRouteTierSampler(NewRegistry()) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + go StartSettingsPoller(ctx, sampler, r, discardLogger()) + + // Initial synchronous poll completes. + synctest.Wait() + require.Equal(t, int32(1), r.getCalls.Load()) + require.True(t, sampler.state.Load().forceFull) + + // Flip the stub to a new value. Advance past the next ticker fire; the synthetic clock advances while everything is + // blocked, the ticker fires, the poller re-polls and applies the new state. Wait then ensures the poller has + // re-blocked before we assert. + r.set(Settings{ + HighVolumeRatio: 0.001, + StandardRatio: 0.02, + ForceFull: false, + }) + time.Sleep(settingsPollInterval + time.Nanosecond) + synctest.Wait() + + require.Equal(t, int32(2), r.getCalls.Load(), "second poll must have fired after one ticker interval") + require.False(t, sampler.state.Load().forceFull, "ticker poll must apply the new state") + + // One more tick with no underlying change should still call Get but should be a no-op for Apply. We verify by the + // invariant that the state is unchanged from the previous assertion. + time.Sleep(settingsPollInterval + time.Nanosecond) + synctest.Wait() + require.Equal(t, int32(3), r.getCalls.Load()) + require.False(t, sampler.state.Load().forceFull) + }) +} diff --git a/server/service/debug_handler.go b/server/service/debug_handler.go index 149e49d57e..9a1f215c35 100644 --- a/server/service/debug_handler.go +++ b/server/service/debug_handler.go @@ -11,6 +11,7 @@ import ( "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/contexts/logging" "github.com/fleetdm/fleet/v4/server/contexts/token" + "github.com/fleetdm/fleet/v4/server/contexts/viewer" "github.com/fleetdm/fleet/v4/server/errorstore" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/service/middleware/auth" @@ -43,7 +44,9 @@ func (m *debugAuthenticationMiddleware) Middleware(next http.Handler) http.Handl return } - next.ServeHTTP(w, r) + // Attach the authenticated viewer to the request context so downstream debug handlers can record who triggered an + // action (e.g. updating trace sampler settings). + next.ServeHTTP(w, r.WithContext(viewer.NewContext(r.Context(), *v))) }) } @@ -92,6 +95,10 @@ func MakeDebugHandler(svc fleet.Service, config config.FleetConfig, logger *slog r.HandleFunc("/debug/db/locks", jsonHandler(logger, func(ctx context.Context) (interface{}, error) { return ds.DBLocks(ctx) })) r.HandleFunc("/debug/db/innodb-status", jsonHandler(logger, func(ctx context.Context) (interface{}, error) { return ds.InnoDBStatus(ctx) })) r.HandleFunc("/debug/db/process-list", jsonHandler(logger, func(ctx context.Context) (interface{}, error) { return ds.ProcessList(ctx) })) + r.HandleFunc("/debug/trace_sampler", jsonHandler(logger, func(ctx context.Context) (any, error) { + return ds.GetTraceSamplerSettings(ctx) + })).Methods(http.MethodGet) + r.HandleFunc("/debug/trace_sampler", patchTraceSamplerHandler(logger, ds)).Methods(http.MethodPatch) mw := &debugAuthenticationMiddleware{ service: svc, diff --git a/server/service/debug_trace_sampler.go b/server/service/debug_trace_sampler.go new file mode 100644 index 0000000000..2c3796d683 --- /dev/null +++ b/server/service/debug_trace_sampler.go @@ -0,0 +1,113 @@ +package service + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "time" + + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/contexts/viewer" + "github.com/fleetdm/fleet/v4/server/fleet" +) + +// traceSamplerPatchRequest is the PATCH payload. Fields are pointers so we can distinguish "unset" from a zero value. PATCH +// semantics mean only the provided fields are applied. +type traceSamplerPatchRequest struct { + HighVolumeRatio *float64 `json:"high_volume_ratio,omitempty"` + StandardRatio *float64 `json:"standard_ratio,omitempty"` + ForceFull *bool `json:"force_full,omitempty"` +} + +// patchTraceSamplerHandler returns the PATCH /debug/trace_sampler handler. The GET path is wired separately in +// MakeDebugHandler via the existing jsonHandler helper, matching the convention used by /debug/migrations and /debug/db/*. +// +// PATCH is necessarily bespoke because no other /debug/ endpoint takes a request body. It validates ratios in [0, 1] and +// persists the change. The replica's in memory sampler picks up the change on the next poller tick (default 60s). +func patchTraceSamplerHandler(logger *slog.Logger, ds fleet.Datastore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // The /debug auth middleware always installs the viewer in context. If it is missing here, the middleware was bypassed + // and we should refuse to record a change rather than silently log user_id=0. That value is indistinguishable from a + // real user id of 0 and weakens the audit trail. + v, ok := viewer.FromContext(r.Context()) + if !ok { + handleServerError(w, r, logger, "debug trace_sampler PATCH refused: viewer missing from context", "viewer required", + errors.New("viewer missing from context")) + return + } + + var req traceSamplerPatchRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, fmt.Sprintf("invalid JSON body: %v", err), http.StatusBadRequest) + return + } + + if err := validateTraceSamplerPatch(req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + current, err := ds.GetTraceSamplerSettings(r.Context()) + if err != nil { + handleServerError(w, r, logger, "debug trace_sampler PATCH read-modify failed", "internal error", err) + return + } + + if req.HighVolumeRatio != nil { + current.HighVolumeRatio = *req.HighVolumeRatio + } + if req.StandardRatio != nil { + current.StandardRatio = *req.StandardRatio + } + if req.ForceFull != nil { + current.ForceFull = *req.ForceFull + } + + if err := ds.SetTraceSamplerSettings(r.Context(), current); err != nil { + handleServerError(w, r, logger, "debug trace_sampler PATCH write failed", "internal error", err) + return + } + + logger.InfoContext(r.Context(), "trace sampler settings updated", + "high_volume_ratio", current.HighVolumeRatio, + "standard_ratio", current.StandardRatio, + "force_full", current.ForceFull, + "updated_by_user_id", v.UserID(), + ) + + // Return the updated row so callers can confirm what was applied. Drop UpdatedAt: the row was read before the write, + // so current.UpdatedAt is the pre-write timestamp (stale and confusing). omitzero on the struct tag skips the field + // when zero. Operators who want the post-write timestamp can do a follow-up GET. + current.UpdatedAt = time.Time{} + b, err := json.MarshalIndent(current, "", " ") + if err != nil { + handleServerError(w, r, logger, "debug trace_sampler PATCH encode response failed", "encoding response", err) + return + } + w.Header().Set("Content-Type", "application/json; charset=utf-8") + _, _ = w.Write(b) + } +} + +// handleServerError centralizes the internal-error response used throughout the trace sampler PATCH handler: it logs logMsg +// with err, records err on the context for the error-handling middleware, and writes clientMsg to the client as a 500. +func handleServerError(w http.ResponseWriter, r *http.Request, logger *slog.Logger, logMsg, clientMsg string, err error) { + logger.ErrorContext(r.Context(), logMsg, "err", err) + ctxerr.Handle(r.Context(), err) + http.Error(w, clientMsg, http.StatusInternalServerError) +} + +func validateTraceSamplerPatch(req traceSamplerPatchRequest) error { + if req.HighVolumeRatio == nil && req.StandardRatio == nil && req.ForceFull == nil { + return errors.New("request body must include at least one of high_volume_ratio, standard_ratio, force_full") + } + if req.HighVolumeRatio != nil && (*req.HighVolumeRatio < 0 || *req.HighVolumeRatio > 1) { + return fmt.Errorf("high_volume_ratio must be in [0, 1], got %v", *req.HighVolumeRatio) + } + if req.StandardRatio != nil && (*req.StandardRatio < 0 || *req.StandardRatio > 1) { + return fmt.Errorf("standard_ratio must be in [0, 1], got %v", *req.StandardRatio) + } + return nil +} diff --git a/server/service/debug_trace_sampler_test.go b/server/service/debug_trace_sampler_test.go new file mode 100644 index 0000000000..c128a80678 --- /dev/null +++ b/server/service/debug_trace_sampler_test.go @@ -0,0 +1,234 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/fleetdm/fleet/v4/server/fleet" + mockds "github.com/fleetdm/fleet/v4/server/mock" + "github.com/fleetdm/fleet/v4/server/platform/tracing" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func discardLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +// adminAuthedRequest builds a request and primes the mockService so the debug auth middleware lets it through as a global +// admin. +func adminAuthedRequest(t *testing.T, method, target string, body string) (*mockService, *http.Request) { + t.Helper() + svc := &mockService{} + svc.On("GetSessionByKey", mock.Anything, "fake_session_key"). + Return(&fleet.Session{UserID: 42, ID: 1}, nil) + svc.On("UserUnauthorized", mock.Anything, uint(42)). + Return(&fleet.User{ID: 42, GlobalRole: new(fleet.RoleAdmin)}, nil) + + var reqBody io.Reader + if body != "" { + reqBody = strings.NewReader(body) + } + req := httptest.NewRequest(method, target, reqBody) + req.Header.Add("Authorization", "BEARER fake_session_key") + return svc, req +} + +func TestTraceSamplerHandler_GET(t *testing.T) { + svc, req := adminAuthedRequest(t, http.MethodGet, "https://fleetdm.com/debug/trace_sampler", "") + + ds := new(mockds.Store) + ds.GetTraceSamplerSettingsFunc = func(_ context.Context) (*tracing.Settings, error) { + return &tracing.Settings{ + HighVolumeRatio: 0.001, + StandardRatio: 0.02, + ForceFull: false, + }, nil + } + + handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds) + res := httptest.NewRecorder() + handler.ServeHTTP(res, req) + + require.Equal(t, http.StatusOK, res.Code) + require.True(t, ds.GetTraceSamplerSettingsFuncInvoked) + + var got tracing.Settings + require.NoError(t, json.Unmarshal(res.Body.Bytes(), &got)) + require.InDelta(t, 0.001, got.HighVolumeRatio, 1e-9) + require.InDelta(t, 0.02, got.StandardRatio, 1e-9) + require.False(t, got.ForceFull) +} + +func TestTraceSamplerHandler_PATCH_PersistsChangesAndReturnsRow(t *testing.T) { + svc, req := adminAuthedRequest(t, http.MethodPatch, + "https://fleetdm.com/debug/trace_sampler", + `{"force_full": true}`) + + ds := new(mockds.Store) + ds.GetTraceSamplerSettingsFunc = func(_ context.Context) (*tracing.Settings, error) { + return &tracing.Settings{ + HighVolumeRatio: 0.001, + StandardRatio: 0.02, + ForceFull: false, + }, nil + } + var saved *tracing.Settings + ds.SetTraceSamplerSettingsFunc = func(_ context.Context, s *tracing.Settings) error { + saved = s + return nil + } + + handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds) + res := httptest.NewRecorder() + handler.ServeHTTP(res, req) + + require.Equal(t, http.StatusOK, res.Code, "PATCH should return 200, body=%s", res.Body.String()) + require.True(t, ds.SetTraceSamplerSettingsFuncInvoked) + require.NotNil(t, saved) + require.True(t, saved.ForceFull, "force_full should now be true") + require.InDelta(t, 0.001, saved.HighVolumeRatio, 1e-9, "other fields should be preserved") + + // Verify the response body matches what was saved. If we forgot to write the response, the test would still see 200 from + // httptest's default but the body would be empty. + var returned tracing.Settings + require.NoError(t, json.Unmarshal(res.Body.Bytes(), &returned)) + require.True(t, returned.ForceFull) + require.InDelta(t, saved.HighVolumeRatio, returned.HighVolumeRatio, 1e-9) + require.InDelta(t, saved.StandardRatio, returned.StandardRatio, 1e-9) + + // PATCH response must NOT include updated_at. The handler reads the row before the write, so the pre-write timestamp + // would be stale. Operators do a follow-up GET to see the post-write value. + require.NotContains(t, res.Body.String(), "updated_at", + "PATCH response must drop updated_at to avoid returning a stale timestamp") +} + +func TestTraceSamplerHandler_PATCH_PartialUpdatePreservesOtherFields(t *testing.T) { + // Locks in the docstring claim that "PATCH semantics mean only the provided fields are applied." Sending only + // high_volume_ratio must leave standard_ratio and force_full at their prior values. + svc, req := adminAuthedRequest(t, http.MethodPatch, + "https://fleetdm.com/debug/trace_sampler", + `{"high_volume_ratio": 0.5}`) + + ds := new(mockds.Store) + ds.GetTraceSamplerSettingsFunc = func(_ context.Context) (*tracing.Settings, error) { + return &tracing.Settings{ + HighVolumeRatio: 0.001, + StandardRatio: 0.07, + ForceFull: true, + }, nil + } + var saved *tracing.Settings + ds.SetTraceSamplerSettingsFunc = func(_ context.Context, s *tracing.Settings) error { + saved = s + return nil + } + + handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds) + res := httptest.NewRecorder() + handler.ServeHTTP(res, req) + + require.Equal(t, http.StatusOK, res.Code, "body=%s", res.Body.String()) + require.NotNil(t, saved) + require.InDelta(t, 0.5, saved.HighVolumeRatio, 1e-9, "high_volume_ratio should be applied") + require.InDelta(t, 0.07, saved.StandardRatio, 1e-9, "standard_ratio should be preserved from the prior row") + require.True(t, saved.ForceFull, "force_full should be preserved from the prior row") +} + +func TestTraceSamplerHandler_PATCH_ReadFailureReturns500(t *testing.T) { + svc, req := adminAuthedRequest(t, http.MethodPatch, + "https://fleetdm.com/debug/trace_sampler", + `{"force_full": true}`) + + ds := new(mockds.Store) + ds.GetTraceSamplerSettingsFunc = func(_ context.Context) (*tracing.Settings, error) { + return nil, errors.New("db unavailable") + } + + handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds) + res := httptest.NewRecorder() + handler.ServeHTTP(res, req) + + require.Equal(t, http.StatusInternalServerError, res.Code) + require.False(t, ds.SetTraceSamplerSettingsFuncInvoked, "should not attempt to write when read fails") +} + +func TestTraceSamplerHandler_PATCH_WriteFailureReturns500(t *testing.T) { + svc, req := adminAuthedRequest(t, http.MethodPatch, + "https://fleetdm.com/debug/trace_sampler", + `{"force_full": true}`) + + ds := new(mockds.Store) + ds.GetTraceSamplerSettingsFunc = func(_ context.Context) (*tracing.Settings, error) { + return &tracing.Settings{HighVolumeRatio: 0.001, StandardRatio: 0.02}, nil + } + ds.SetTraceSamplerSettingsFunc = func(_ context.Context, _ *tracing.Settings) error { + return errors.New("constraint violation") + } + + handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds) + res := httptest.NewRecorder() + handler.ServeHTTP(res, req) + + require.Equal(t, http.StatusInternalServerError, res.Code) + require.True(t, ds.SetTraceSamplerSettingsFuncInvoked) +} + +func TestTraceSamplerHandler_PATCH_RejectsBadJSON(t *testing.T) { + svc, req := adminAuthedRequest(t, http.MethodPatch, + "https://fleetdm.com/debug/trace_sampler", + `{"force_full":`) // malformed + + ds := new(mockds.Store) + handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds) + res := httptest.NewRecorder() + handler.ServeHTTP(res, req) + + require.Equal(t, http.StatusBadRequest, res.Code) + require.False(t, ds.SetTraceSamplerSettingsFuncInvoked) +} + +func TestTraceSamplerHandler_PATCH_RejectsOutOfRangeRatio(t *testing.T) { + cases := []struct { + name string + body string + }{ + {"high above 1", `{"high_volume_ratio": 1.5}`}, + {"high below 0", `{"high_volume_ratio": -0.1}`}, + {"standard above 1", `{"standard_ratio": 2.0}`}, + {"standard below 0", `{"standard_ratio": -1.0}`}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + svc, req := adminAuthedRequest(t, http.MethodPatch, + "https://fleetdm.com/debug/trace_sampler", c.body) + ds := new(mockds.Store) + handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds) + res := httptest.NewRecorder() + handler.ServeHTTP(res, req) + + require.Equal(t, http.StatusBadRequest, res.Code) + require.Contains(t, res.Body.String(), "must be in [0, 1]") + require.False(t, ds.SetTraceSamplerSettingsFuncInvoked) + }) + } +} + +func TestTraceSamplerHandler_PATCH_RequiresAtLeastOneField(t *testing.T) { + svc, req := adminAuthedRequest(t, http.MethodPatch, + "https://fleetdm.com/debug/trace_sampler", `{}`) + ds := new(mockds.Store) + handler := MakeDebugHandler(svc, testConfig, discardLogger(), nil, ds) + res := httptest.NewRecorder() + handler.ServeHTTP(res, req) + + require.Equal(t, http.StatusBadRequest, res.Code) + require.Contains(t, res.Body.String(), "at least one") +} diff --git a/server/service/tracing_tiers.go b/server/service/tracing_tiers.go new file mode 100644 index 0000000000..200739d51a --- /dev/null +++ b/server/service/tracing_tiers.go @@ -0,0 +1,62 @@ +package service + +import ( + "net/http" + + "github.com/fleetdm/fleet/v4/server/platform/tracing" +) + +// RegisterTracingTiers populates the trace sampling registry with the tier classifications for routes owned by this package +// (the legacy flat server/service handler). +// +// Bounded contexts that have moved out of this package (e.g. server/activity/) own their own registrations. They +// should expose a RegisterTracingTiers function the serve command calls during startup. The platform/tracing package itself +// stays free of route knowledge. +// +// Paths use "_version_" as the placeholder for the gorilla/mux fleetversion segment. The sampler normalizes incoming span +// names back to that form before lookup. Alternate path forms (e.g. /api/v1/osquery/...) are registered separately. +// +// Unregistered routes (including all cron jobs, enroll, SCEP, MDM checkin, command ack/result, GitOps batch, etc.) fall to +// TierAlways by design. +func RegisterTracingTiers(registry *tracing.Registry) { + // Hot agent endpoints. These dominate request volume at scale (tens of thousands of spans per second on a 100k-host + // fleet) without being individually interesting. Sampled at the configured high volume ratio (default 0.1%). + registry.Register(http.MethodPost, "/api/osquery/config", tracing.TierHighVolume) + registry.Register(http.MethodPost, "/api/v1/osquery/config", tracing.TierHighVolume) + registry.Register(http.MethodPost, "/api/osquery/distributed/read", tracing.TierHighVolume) + registry.Register(http.MethodPost, "/api/v1/osquery/distributed/read", tracing.TierHighVolume) + registry.Register(http.MethodPost, "/api/osquery/distributed/write", tracing.TierHighVolume) + registry.Register(http.MethodPost, "/api/v1/osquery/distributed/write", tracing.TierHighVolume) + registry.Register(http.MethodPost, "/api/osquery/log", tracing.TierHighVolume) + registry.Register(http.MethodPost, "/api/v1/osquery/log", tracing.TierHighVolume) + registry.Register(http.MethodPost, "/api/fleet/orbit/config", tracing.TierHighVolume) + registry.Register(http.MethodPost, "/api/fleet/orbit/device_token", tracing.TierHighVolume) + registry.Register(http.MethodHead, "/api/fleet/orbit/ping", tracing.TierHighVolume) + registry.Register(http.MethodHead, "/api/fleet/device/ping", tracing.TierHighVolume) + registry.Register(http.MethodHead, "/api/_version_/fleet/device/{token}/ping", tracing.TierHighVolume) + registry.Register(http.MethodGet, "/api/_version_/fleet/device/{token}/desktop", tracing.TierHighVolume) + + // Admin / read endpoints. Moderate volume, moderate diagnostic value. Sampled at the configured standard ratio (default + // 2%). Starting set covers the highest traffic admin reads plus the per page load endpoints (config, me, version, etc.) + // that the UI hits on every navigation. Expand as we observe traffic patterns in dogfood and the customer pilot. + registry.Register(http.MethodGet, "/api/_version_/fleet/config", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/fleets", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/host_summary", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/hosts", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/hosts/{id}", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/hosts/count", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/hosts/identifier/{identifier}", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/hosts/summary/mdm", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/labels", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/me", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/policies", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/reports", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/software", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/software/titles", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/software/versions", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/spec/enroll_secret", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/users", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/version", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/charts/{metric}", tracing.TierStandard) + registry.Register(http.MethodGet, "/api/_version_/fleet/android_enterprise", tracing.TierStandard) +}