Discharge7443/fleet
1
0
Fork 0
Code Issues Pull Requests Actions 8 Packages Projects Releases Wiki Activity

Signoz action fixes (#38656)

Browse Source
This commit is contained in:
Robert Fairburn
2026-01-22 19:10:44 -06:00
committed by GitHub
parent 1a382a5b36
commit ffe6df25be
5 changed files with 166 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/dogfood-signoz-deploy.yml
+1 -6
View File
@@ -3,10 +3,6 @@ name: Deploy Dogfood SigNoz
on:
workflow_dispatch:
inputs:
workspace:
description: "Terraform workspace to deploy SigNoz into."
required: true
default: "fleet"
dry_run:
description: Dry run only? No "terraform apply"
type: boolean
@@ -25,7 +21,6 @@ env:
AWS_REGION: us-east-2
AWS_IAM_ROLE: arn:aws:iam::160035666661:role/github-actions-role
TF_ACTIONS_WORKING_DIR: infrastructure/dogfood/terraform/aws-tf-module/signoz
TF_WORKSPACE: ${{ github.event.inputs.workspace }}
TF_VAR_otel_bearer_token: ${{ secrets.DOGFOOD_SIGNOZ_OTEL_BEARER_TOKEN }}
permissions:
@@ -58,7 +53,7 @@ jobs:
terraform_wrapper: false
- name: Set up kubectl
uses: azure/setup-kubectl@18a212d6b26e63d2e08d083e82f0597a6095f08c # v4.0.0
uses: azure/setup-kubectl@776406bce94f63e41d621b960d78ee25c8b76ede # v4
with:
version: v1.31.0
infrastructure/dogfood/terraform/aws-tf-module/github.tf
+8
View File
@@ -105,4 +105,12 @@ data "aws_iam_policy_document" "gha-permissions" {
]
resources = ["*"]
}
statement {
effect = "Allow"
actions = [
"sts:AssumeRole"
]
resources = ["arn:aws:iam::353365949058:role/terraform-dogfood"]
}
}
infrastructure/dogfood/terraform/aws-tf-module/signoz/main.tf
+49 -2
View File
@@ -73,6 +73,8 @@ data "aws_eks_cluster_auth" "signoz" {
name = module.eks.cluster_name
}
data "aws_caller_identity" "current" {}
data "aws_route53_zone" "dogfood" {
name = "dogfood.fleetdm.com."
}
@@ -83,6 +85,9 @@ locals {
otlp_domain = "otlp.signoz.dogfood.fleetdm.com"
signoz_alb_name = "signoz-dogfood"
alb_subnets = join(",", compact(data.terraform_remote_state.dogfood.outputs.vpc.private_subnets))
admin_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/admin"
github_actions_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/github-actions-role"
sso_admin_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-reserved/sso.amazonaws.com/us-east-2/AWSReservedSSO_DogfoodAdministrators_646894fd206a2661"
common_ingress_annotations = {
"alb.ingress.kubernetes.io/scheme" = "internal"
"alb.ingress.kubernetes.io/subnets" = local.alb_subnets
@@ -152,8 +157,50 @@ module "eks" {
}
}
# Enable cluster creator admin access
enable_cluster_creator_admin_permissions = true
# Use explicit access entries to avoid creator-role drift.
enable_cluster_creator_admin_permissions = false
kms_key_administrators = [
local.admin_role_arn,
local.github_actions_role_arn,
local.sso_admin_role_arn
]
access_entries = {
admin = {
principal_arn = local.admin_role_arn
policy_associations = {
admin = {
policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"
access_scope = {
type = "cluster"
}
}
}
}
github_actions = {
principal_arn = local.github_actions_role_arn
policy_associations = {
admin = {
policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"
access_scope = {
type = "cluster"
}
}
}
}
sso_admin = {
principal_arn = local.sso_admin_role_arn
policy_associations = {
admin = {
policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"
access_scope = {
type = "cluster"
}
}
}
}
}
# Enable OIDC provider for IRSA (IAM Roles for Service Accounts)
enable_irsa = true
infrastructure/dogfood/terraform/aws-tf-module/signoz/otel-collector-values.yaml
+103 -45
View File
@@ -7,39 +7,75 @@ otelCollector:
lowCardinalityExceptionGrouping: true
config:
extensions:
health_check:
endpoint: 0.0.0.0:13133
bearertokenauth:
scheme: "Bearer"
token: ${env:OTEL_BEARER_TOKEN}
connectors:
signozmeter:
metrics_flush_interval: 1h
dimensions:
- name: service.name
- name: deployment.environment
- name: host.name
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
max_recv_msg_size_mib: 16
auth:
authenticator: bearertokenauth
http:
endpoint: 0.0.0.0:4318
auth:
authenticator: bearertokenauth
jaeger:
protocols:
grpc:
endpoint: 0.0.0.0:14250
thrift_http:
endpoint: 0.0.0.0:14268
httplogreceiver/heroku:
endpoint: 0.0.0.0:8081
source: heroku
httplogreceiver/json:
endpoint: 0.0.0.0:8082
source: json
processors:
batch:
send_batch_size: 2000
timeout: 5s
batch/meter:
send_batch_max_size: 25000
send_batch_size: 20000
timeout: 1s
memory_limiter:
check_interval: 1s
limit_mib: 11264
spike_limit_mib: 512
signozspanmetrics/delta:
metrics_exporter: signozclickhousemetrics
latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s]
dimensions_cache_size: 100000
dimensions:
- name: service.namespace
default: default
- name: deployment.environment
default: default
- name: signoz.collector.id
aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA
extensions:
health_check:
endpoint: 0.0.0.0:13133
zpages:
endpoint: localhost:55679
pprof:
endpoint: localhost:1777
bearertokenauth:
scheme: "Bearer"
token: ${env:OTEL_BEARER_TOKEN}
exporters:
# Logs exporter with balanced queue sizes for throughput and memory
clickhouselogsexporter:
timeout: 60s
retry_on_failure:
enabled: true
initial_interval: 5s
max_interval: 30s
max_elapsed_time: 300s
sending_queue:
enabled: true
num_consumers: 10
queue_size: 10000
# Traces exporter must handle spans after tail_sampling
clickhousetraces:
datasource: tcp://${env:CLICKHOUSE_USER}:${env:CLICKHOUSE_PASSWORD}@${env:CLICKHOUSE_HOST}:${env:CLICKHOUSE_PORT}/${env:CLICKHOUSE_TRACE_DATABASE}
low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING}
use_new_schema: true
timeout: 120s
retry_on_failure:
enabled: true
@@ -50,35 +86,57 @@ otelCollector:
enabled: true
num_consumers: 10
queue_size: 10000
# Metadata exporter with increased timeout and retry support
metadataexporter:
signozclickhousemetrics:
dsn: tcp://${env:CLICKHOUSE_USER}:${env:CLICKHOUSE_PASSWORD}@${env:CLICKHOUSE_HOST}:${env:CLICKHOUSE_PORT}/${env:CLICKHOUSE_DATABASE}
timeout: 45s
clickhouselogsexporter:
dsn: tcp://${env:CLICKHOUSE_USER}:${env:CLICKHOUSE_PASSWORD}@${env:CLICKHOUSE_HOST}:${env:CLICKHOUSE_PORT}/${env:CLICKHOUSE_LOG_DATABASE}
timeout: 60s
use_new_schema: true
retry_on_failure:
enabled: true
initial_interval: 5s
max_interval: 30s
max_elapsed_time: 300s
processors:
# Optimized batch processor to send data more frequently
# Smaller batches reduce memory pressure
batch:
send_batch_size: 2000
timeout: 5s
# Memory limiter to prevent out-of-memory by applying back pressure
memory_limiter:
check_interval: 1s
limit_mib: 11264
spike_limit_mib: 512
sending_queue:
enabled: true
num_consumers: 10
queue_size: 10000
metadataexporter:
dsn: tcp://${env:CLICKHOUSE_USER}:${env:CLICKHOUSE_PASSWORD}@${env:CLICKHOUSE_HOST}:${env:CLICKHOUSE_PORT}/signoz_metadata
timeout: 60s
tenant_id: ${env:TENANT_ID}
cache:
provider: in_memory
retry_on_failure:
enabled: true
initial_interval: 5s
max_interval: 30s
max_elapsed_time: 300s
signozclickhousemeter:
dsn: tcp://${env:CLICKHOUSE_USER}:${env:CLICKHOUSE_PASSWORD}@${env:CLICKHOUSE_HOST}:${env:CLICKHOUSE_PORT}/${env:CLICKHOUSE_METER_DATABASE}
timeout: 45s
sending_queue:
enabled: false
service:
extensions:
- health_check
- bearertokenauth
telemetry:
logs:
encoding: json
extensions: [health_check, zpages, pprof, bearertokenauth]
pipelines:
traces:
processors:
- memory_limiter
- batch
receivers: [otlp, jaeger]
processors: [memory_limiter, signozspanmetrics/delta, batch]
exporters: [clickhousetraces, metadataexporter, signozmeter]
metrics:
receivers: [otlp]
processors: [batch]
exporters: [metadataexporter, signozclickhousemetrics, signozmeter]
logs:
receivers: [otlp, httplogreceiver/heroku, httplogreceiver/json]
processors: [batch]
exporters: [clickhouselogsexporter, metadataexporter, signozmeter]
metrics/meter:
receivers: [signozmeter]
processors: [batch/meter]
exporters: [signozclickhousemeter]
infrastructure/dogfood/terraform/aws-tf-module/signoz/variables.tf
+5
View File
@@ -0,0 +1,5 @@
variable "otel_bearer_token" {
type = string
sensitive = true
description = "Bearer token required by the SigNoz OTLP collector."
}