Improved OpenTelemetry tracing (#38483)
<!-- Add the related story/sub-task/bug number, like Resolves #123, or remove if NA --> **Related issue:** Resolves #38479 # Checklist for submitter - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. ## Testing - [x] QA'd all new/changed functionality manually
This commit is contained in:
@@ -0,0 +1 @@
|
||||
Improved OpenTelemetry tracing: added proper shutdown to flush pending spans, and added service name/version resource attributes for better trace identification.
|
||||
+23
-1
@@ -94,7 +94,9 @@ import (
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.34.0"
|
||||
"google.golang.org/grpc"
|
||||
_ "google.golang.org/grpc/encoding/gzip" // Because we use gzip compression for OTLP
|
||||
)
|
||||
@@ -158,6 +160,7 @@ the way that the Fleet server works.
|
||||
}
|
||||
|
||||
// Init tracing
|
||||
var tracerProvider *sdktrace.TracerProvider
|
||||
if config.Logging.TracingEnabled {
|
||||
ctx := context.Background()
|
||||
client := otlptracegrpc.NewClient(
|
||||
@@ -172,7 +175,20 @@ the way that the Fleet server works.
|
||||
batchSpanProcessor := sdktrace.NewBatchSpanProcessor(otlpTraceExporter,
|
||||
sdktrace.WithMaxExportBatchSize(256), // Reduce from default 512 to 256
|
||||
)
|
||||
tracerProvider := sdktrace.NewTracerProvider(
|
||||
// Create resource with service identification attributes
|
||||
res, err := resource.Merge(
|
||||
resource.Default(),
|
||||
resource.NewWithAttributes(
|
||||
semconv.SchemaURL,
|
||||
semconv.ServiceName("fleet"),
|
||||
semconv.ServiceVersion(version.Version().Version),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
initFatal(err, "Failed to create OTEL resource")
|
||||
}
|
||||
tracerProvider = sdktrace.NewTracerProvider(
|
||||
sdktrace.WithResource(res),
|
||||
sdktrace.WithSpanProcessor(batchSpanProcessor),
|
||||
)
|
||||
otel.SetTracerProvider(tracerProvider)
|
||||
@@ -1646,6 +1662,12 @@ the way that the Fleet server works.
|
||||
cancelFunc()
|
||||
cleanupCronStatsOnShutdown(ctx, ds, logger, instanceID)
|
||||
launcher.GracefulStop()
|
||||
// Flush any pending OTEL spans before shutting down
|
||||
if tracerProvider != nil {
|
||||
if err := tracerProvider.Shutdown(ctx); err != nil {
|
||||
level.Error(logger).Log("msg", "failed to shutdown OTEL tracer provider", "err", err)
|
||||
}
|
||||
}
|
||||
return srv.Shutdown(ctx)
|
||||
}()
|
||||
}()
|
||||
|
||||
Reference in New Issue
Block a user