From e0631aff7617fd97661c0613ff013e063ef9227b Mon Sep 17 00:00:00 2001 From: Robert Fairburn <8029478+rfairburn@users.noreply.github.com> Date: Thu, 22 Jan 2026 12:33:27 -0600 Subject: [PATCH] Dogfood signoz (#38569) --- .github/workflows/dogfood-deploy.yml | 1 + .github/workflows/dogfood-signoz-deploy.yml | 85 +++ .../dogfood/terraform/aws-tf-module/github.tf | 1 + .../dogfood/terraform/aws-tf-module/main.tf | 54 +- .../terraform/aws-tf-module/signoz/main.tf | 569 ++++++++++++++++++ .../signoz/otel-collector-values.yaml | 84 +++ .../terraform/aws-tf-module/signoz/outputs.tf | 28 + .../signoz/scripts/send-test-telemetry.sh | 21 + 8 files changed, 839 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/dogfood-signoz-deploy.yml create mode 100644 infrastructure/dogfood/terraform/aws-tf-module/signoz/main.tf create mode 100644 infrastructure/dogfood/terraform/aws-tf-module/signoz/otel-collector-values.yaml create mode 100644 infrastructure/dogfood/terraform/aws-tf-module/signoz/outputs.tf create mode 100755 infrastructure/dogfood/terraform/aws-tf-module/signoz/scripts/send-test-telemetry.sh diff --git a/.github/workflows/dogfood-deploy.yml b/.github/workflows/dogfood-deploy.yml index 1e48442835..6ee4b0ec5e 100644 --- a/.github/workflows/dogfood-deploy.yml +++ b/.github/workflows/dogfood-deploy.yml @@ -38,6 +38,7 @@ env: TF_VAR_elastic_token: ${{ secrets.ELASTIC_APM_SECRET_TOKEN }} TF_VAR_geolite2_license: ${{ secrets.MAXMIND_LICENSE }} TF_VAR_dogfood_sidecar_enroll_secret: ${{ secrets.DOGFOOD_SERVERS_CANARY_ENROLL_SECRET }} + TF_VAR_otel_bearer_token: ${{ secrets.DOGFOOD_SIGNOZ_OTEL_BEARER_TOKEN }} TF_VAR_android_service_credentials: ${{ secrets.FLEET_DEV_ANDROID_SERVICE_CREDENTIALS }} TF_VAR_cloudfront_public_key: ${{ secrets.CLOUDFRONT_SIGNING_PUBLIC_KEY }} TF_VAR_cloudfront_private_key: ${{ secrets.CLOUDFRONT_SIGNING_PRIVATE_KEY }} diff --git a/.github/workflows/dogfood-signoz-deploy.yml b/.github/workflows/dogfood-signoz-deploy.yml new file mode 100644 index 0000000000..d9e8708f8f --- /dev/null +++ b/.github/workflows/dogfood-signoz-deploy.yml @@ -0,0 +1,85 @@ +name: Deploy Dogfood SigNoz + +on: + workflow_dispatch: + inputs: + workspace: + description: "Terraform workspace to deploy SigNoz into." + required: true + default: "fleet" + dry_run: + description: Dry run only? No "terraform apply" + type: boolean + default: false + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id}} + cancel-in-progress: true + +defaults: + run: + shell: bash + working-directory: infrastructure/dogfood/terraform/aws-tf-module/signoz + +env: + AWS_REGION: us-east-2 + AWS_IAM_ROLE: arn:aws:iam::160035666661:role/github-actions-role + TF_ACTIONS_WORKING_DIR: infrastructure/dogfood/terraform/aws-tf-module/signoz + TF_WORKSPACE: ${{ github.event.inputs.workspace }} + TF_VAR_otel_bearer_token: ${{ secrets.DOGFOOD_SIGNOZ_OTEL_BEARER_TOKEN }} + +permissions: + id-token: write + contents: read + +jobs: + deploy: + name: Deploy Dogfood SigNoz + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 + with: + egress-policy: audit + + - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + - id: fail-on-main + run: "false" + if: ${{ github.ref == 'main' }} + + - uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 # v1.7.0 + with: + role-to-assume: ${{ env.AWS_IAM_ROLE }} + aws-region: ${{ env.AWS_REGION }} + + - uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3 + with: + terraform_version: 1.10.2 + terraform_wrapper: false + + - name: Set up kubectl + uses: azure/setup-kubectl@18a212d6b26e63d2e08d083e82f0597a6095f08c # v4.0.0 + with: + version: v1.31.0 + + - name: Configure kubeconfig + run: aws eks update-kubeconfig --region "${AWS_REGION}" --name signoz + + - name: Terraform Init + id: init + run: terraform init + - name: Terraform fmt + id: fmt + run: terraform fmt -check + continue-on-error: true + - name: Terraform Validate + id: validate + run: terraform validate -no-color + - name: Terraform Plan + id: plan + run: terraform plan -no-color + continue-on-error: true + - name: Terraform Apply + if: inputs.dry_run == false + id: apply + run: terraform apply -auto-approve diff --git a/infrastructure/dogfood/terraform/aws-tf-module/github.tf b/infrastructure/dogfood/terraform/aws-tf-module/github.tf index 5d12b134a6..f7664c308b 100644 --- a/infrastructure/dogfood/terraform/aws-tf-module/github.tf +++ b/infrastructure/dogfood/terraform/aws-tf-module/github.tf @@ -70,6 +70,7 @@ data "aws_iam_policy_document" "gha-permissions" { "s3:*", "lambda:*", "ecs:*", + "eks:*", "rds:*", "rds-data:*", "secretsmanager:*", diff --git a/infrastructure/dogfood/terraform/aws-tf-module/main.tf b/infrastructure/dogfood/terraform/aws-tf-module/main.tf index 389568b12b..06f29ff413 100644 --- a/infrastructure/dogfood/terraform/aws-tf-module/main.tf +++ b/infrastructure/dogfood/terraform/aws-tf-module/main.tf @@ -47,6 +47,10 @@ variable "cloudfront_private_key" {} variable "webhook_url" { description = "Webhook URL used for Webhook Logging Destination" } +variable "otel_bearer_token" { + sensitive = true + description = "Bearer token for OTLP ingest into SigNoz." +} data "aws_caller_identity" "current" {} @@ -55,9 +59,13 @@ locals { fleet_image = var.fleet_image # Set this to the version of fleet to be deployed geolite2_image = "${aws_ecr_repository.fleet.repository_url}:${split(":", var.fleet_image)[1]}-geolite2-${formatdate("YYYYMMDDhhmm", timestamp())}" extra_environment_variables = { - FLEET_LICENSE_KEY = var.fleet_license - FLEET_LOGGING_DEBUG = "true" - FLEET_LOGGING_JSON = "true" + FLEET_LICENSE_KEY = var.fleet_license + FLEET_LOGGING_DEBUG = "true" + FLEET_LOGGING_JSON = "true" + FLEET_LOGGING_TRACING_ENABLED = "true" + FLEET_LOGGING_TRACING_TYPE = "opentelemetry" + OTEL_SERVICE_NAME = local.customer + OTEL_EXPORTER_OTLP_ENDPOINT = "https://otlp.signoz.dogfood.fleetdm.com" # FLEET_LOGGING_TRACING_ENABLED = "true" # FLEET_LOGGING_TRACING_TYPE = "elasticapm" FLEET_MYSQL_MAX_OPEN_CONNS = "10" @@ -85,6 +93,9 @@ locals { sentry_secrets = { FLEET_SENTRY_DSN = "${aws_secretsmanager_secret.sentry.arn}:FLEET_SENTRY_DSN::" } + signoz_secrets = { + OTEL_EXPORTER_OTLP_HEADERS = "${aws_secretsmanager_secret.signoz_otel_bearer_token.arn}:OTEL_EXPORTER_OTLP_HEADERS::" + } # idp_metadata_file = "${path.module}/files/idp-metadata.xml" /* @@ -201,13 +212,19 @@ module "main" { ) extra_execution_iam_policies = concat( module.mdm.extra_execution_iam_policies, - [aws_iam_policy.sentry.arn, aws_iam_policy.osquery_sidecar.arn, aws_iam_policy.entra_conditional_access.arn], + [ + aws_iam_policy.sentry.arn, + aws_iam_policy.osquery_sidecar.arn, + aws_iam_policy.entra_conditional_access.arn, + aws_iam_policy.signoz_otel_bearer_token.arn + ], module.cloudfront-software-installers.extra_execution_iam_policies, ) #, module.saml_auth_proxy.fleet_extra_execution_policies) extra_secrets = merge( local.entra_conditional_access_secrets, module.mdm.extra_secrets, local.sentry_secrets, + local.signoz_secrets, module.cloudfront-software-installers.extra_secrets ) private_key_secret_name = "${local.customer}-fleet-server-private-key" @@ -427,6 +444,31 @@ data "aws_iam_policy_document" "sentry" { } } +resource "aws_secretsmanager_secret" "signoz_otel_bearer_token" { + name = "${local.customer}-signoz-otel-bearer-token" +} + +resource "aws_secretsmanager_secret_version" "signoz_otel_bearer_token" { + secret_id = aws_secretsmanager_secret.signoz_otel_bearer_token.id + secret_string = jsonencode({ + OTEL_EXPORTER_OTLP_HEADERS = "Authorization=Bearer ${var.otel_bearer_token}" + }) +} + +resource "aws_iam_policy" "signoz_otel_bearer_token" { + name = "fleet-signoz-otel-bearer-token-policy" + policy = data.aws_iam_policy_document.signoz_otel_bearer_token.json +} + +data "aws_iam_policy_document" "signoz_otel_bearer_token" { + statement { + actions = [ + "secretsmanager:GetSecretValue", + ] + resources = [aws_secretsmanager_secret.signoz_otel_bearer_token.arn] + } +} + module "migrations" { depends_on = [ module.geolite2 @@ -761,3 +803,7 @@ module "cloudfront-software-installers" { enable_logging = true logging_s3_bucket = module.logging_alb.log_s3_bucket_id } + +output "vpc" { + value = module.main.vpc +} diff --git a/infrastructure/dogfood/terraform/aws-tf-module/signoz/main.tf b/infrastructure/dogfood/terraform/aws-tf-module/signoz/main.tf new file mode 100644 index 0000000000..bf7201877e --- /dev/null +++ b/infrastructure/dogfood/terraform/aws-tf-module/signoz/main.tf @@ -0,0 +1,569 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.68.0" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.11" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.23" + } + } + + backend "s3" { + bucket = "fleet-terraform-state20220408141538466600000002" + key = "dogfood/signoz/terraform.tfstate" + workspace_key_prefix = "dogfood" + region = "us-east-2" + encrypt = true + kms_key_id = "9f98a443-ffd7-4dbe-a9c3-37df89b2e42a" + dynamodb_table = "tf-remote-state-lock" + assume_role = { + role_arn = "arn:aws:iam::353365949058:role/terraform-dogfood" + } + } +} + +provider "aws" { + default_tags { + tags = { + environment = "dogfood-signoz" + terraform = "https://github.com/fleetdm/fleet/tree/main/infrastructure/dogfood/terraform/aws-tf-module/signoz" + workspace = terraform.workspace + } + } +} + +# Read shared VPC from remote state +data "terraform_remote_state" "dogfood" { + backend = "s3" + workspace = "fleet" + config = { + bucket = "fleet-terraform-remote-state" + key = "fleet" + region = "us-east-2" + dynamodb_table = "fleet-terraform-state-lock" + } +} + +# Providers for Helm/Kubernetes after cluster is created +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.signoz.token + } +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.signoz.token +} + +data "aws_region" "current" {} + +data "aws_eks_cluster_auth" "signoz" { + name = module.eks.cluster_name +} + +data "aws_route53_zone" "dogfood" { + name = "dogfood.fleetdm.com." +} + +locals { + cluster_name = "signoz" + signoz_domain = "signoz.dogfood.fleetdm.com" + otlp_domain = "otlp.signoz.dogfood.fleetdm.com" + signoz_alb_name = "signoz-dogfood" + alb_subnets = join(",", compact(data.terraform_remote_state.dogfood.outputs.vpc.private_subnets)) + common_ingress_annotations = { + "alb.ingress.kubernetes.io/scheme" = "internal" + "alb.ingress.kubernetes.io/subnets" = local.alb_subnets + "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTPS\":443}]" + "alb.ingress.kubernetes.io/certificate-arn" = module.acm.acm_certificate_arn + "alb.ingress.kubernetes.io/backend-protocol" = "HTTP" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/group.name" = local.signoz_alb_name + "alb.ingress.kubernetes.io/load-balancer-name" = local.signoz_alb_name + } + signoz_ingress_annotations = local.common_ingress_annotations + otlp_ingress_annotations = merge(local.common_ingress_annotations, { + "alb.ingress.kubernetes.io/backend-protocol-version" = "GRPC" + }) +} + +module "acm" { + source = "terraform-aws-modules/acm/aws" + version = "4.3.1" + + domain_name = local.signoz_domain + subject_alternative_names = [local.otlp_domain] + zone_id = data.aws_route53_zone.dogfood.zone_id + + wait_for_validation = true +} + +# Use shared fleet VPC +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 21.0" + + name = local.cluster_name + kubernetes_version = "1.31" + + endpoint_public_access = true + + vpc_id = data.terraform_remote_state.dogfood.outputs.vpc.vpc_id + subnet_ids = data.terraform_remote_state.dogfood.outputs.vpc.private_subnets + + # IMPORTANT: Install critical addons BEFORE node group to avoid circular dependency + # Nodes need VPC CNI to become Ready, but terraform waits for nodes to be Ready + # before creating addons. This causes a deadlock where nodes are stuck NotReady. + # Solution: Use addons with before_compute=true to install VPC CNI before node group completes. + addons = { + vpc-cni = { + most_recent = true + before_compute = true + } + kube-proxy = { + most_recent = true + before_compute = true + } + coredns = { + most_recent = true + before_compute = true + } + } + + # Managed node group + eks_managed_node_groups = { + default = { + min_size = 2 + max_size = 3 + desired_size = 2 + instance_types = ["t3.xlarge"] + } + } + + # Enable cluster creator admin access + enable_cluster_creator_admin_permissions = true + + # Enable OIDC provider for IRSA (IAM Roles for Service Accounts) + enable_irsa = true +} + +# IAM Role for EBS CSI Driver Service Account (IRSA) +module "ebs_csi_irsa_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.0" + + role_name = "${local.cluster_name}-ebs-csi-driver" + attach_ebs_csi_policy = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"] + } + } +} + +# IAM Role for AWS Load Balancer Controller Service Account (IRSA) +module "load_balancer_controller_irsa_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.0" + + role_name = "${local.cluster_name}-aws-load-balancer-controller" + + attach_load_balancer_controller_policy = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-load-balancer-controller"] + } + } +} + +# EBS CSI Driver addon with IRSA support +# This must be created separately to avoid circular dependency with OIDC provider +resource "aws_eks_addon" "ebs_csi" { + cluster_name = module.eks.cluster_name + addon_name = "aws-ebs-csi-driver" + addon_version = data.aws_eks_addon_version.ebs_csi.version + service_account_role_arn = module.ebs_csi_irsa_role.iam_role_arn + + depends_on = [ + module.ebs_csi_irsa_role, + module.eks + ] +} + +data "aws_eks_addon_version" "ebs_csi" { + addon_name = "aws-ebs-csi-driver" + kubernetes_version = module.eks.cluster_version + most_recent = true +} + +# Wait for EBS CSI driver to be active +resource "time_sleep" "wait_for_ebs_csi" { + depends_on = [aws_eks_addon.ebs_csi] + + create_duration = "60s" +} + +# Create gp3 storage class using EBS CSI driver +resource "kubernetes_storage_class_v1" "gp3" { + metadata { + name = "gp3" + annotations = { + "storageclass.kubernetes.io/is-default-class" = "true" + } + } + + storage_provisioner = "ebs.csi.aws.com" + reclaim_policy = "Delete" + allow_volume_expansion = true + volume_binding_mode = "WaitForFirstConsumer" + + parameters = { + type = "gp3" + encrypted = "true" + fsType = "ext4" + } + + depends_on = [time_sleep.wait_for_ebs_csi] +} + +# AWS Load Balancer Controller via Helm (managed add-on not available for 1.31 in all regions) +resource "helm_release" "aws_load_balancer_controller" { + name = "aws-load-balancer-controller" + repository = "https://aws.github.io/eks-charts" + chart = "aws-load-balancer-controller" + namespace = "kube-system" + + wait = true + wait_for_jobs = false + + set { + name = "clusterName" + value = module.eks.cluster_name + } + + set { + name = "region" + value = data.aws_region.current.region + } + + set { + name = "vpcId" + value = data.terraform_remote_state.dogfood.outputs.vpc.vpc_id + } + + set { + name = "serviceAccount.create" + value = "true" + } + + set { + name = "serviceAccount.name" + value = "aws-load-balancer-controller" + } + + set { + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.load_balancer_controller_irsa_role.iam_role_arn + } + + depends_on = [ + module.eks, + module.load_balancer_controller_irsa_role + ] +} + +# SigNoz via Helm +resource "helm_release" "signoz" { + name = "signoz" + repository = "https://charts.signoz.io" + chart = "signoz" + namespace = "signoz" + timeout = 3600 # 60 minutes - allows for worst-case AWS delays during both apply and destroy + + create_namespace = true + wait = true + wait_for_jobs = false + + # OTEL Collector configuration overrides for production stability + values = [ + file("${path.module}/otel-collector-values.yaml"), + yamlencode({ + signoz = { + ingress = { + annotations = local.signoz_ingress_annotations + } + } + otelCollector = { + ingress = { + annotations = local.otlp_ingress_annotations + } + } + }) + ] + + set { + name = "cloud" + value = "false" + } + + set { + name = "signoz.service.type" + value = "ClusterIP" + } + + set { + name = "signoz.ingress.enabled" + value = "true" + } + + set { + name = "signoz.ingress.className" + value = "alb" + } + + set { + name = "signoz.ingress.hosts[0].host" + value = local.signoz_domain + } + + set { + name = "signoz.ingress.hosts[0].paths[0].path" + value = "/*" + } + + set { + name = "signoz.ingress.hosts[0].paths[0].pathType" + value = "ImplementationSpecific" + } + + set { + name = "signoz.ingress.hosts[0].paths[0].port" + value = "8080" + } + + # OTLP collector should be internal only (not publicly accessible) + set { + name = "otelCollector.service.type" + value = "ClusterIP" + } + + set { + name = "otelCollector.ingress.enabled" + value = "true" + } + + set { + name = "otelCollector.ingress.className" + value = "alb" + } + + set { + name = "otelCollector.ingress.hosts[0].host" + value = local.otlp_domain + } + + set { + name = "otelCollector.ingress.hosts[0].paths[0].path" + value = "/*" + } + + set { + name = "otelCollector.ingress.hosts[0].paths[0].pathType" + value = "ImplementationSpecific" + } + + set { + name = "otelCollector.ingress.hosts[0].paths[0].port" + value = "4317" + } + + + # Clickhouse storage configuration + set { + name = "clickhouse.persistence.size" + value = "200Gi" + } + + set { + name = "clickhouse.persistence.storageClass" + value = "gp3" + } + + # Zookeeper storage configuration + set { + name = "zookeeper.persistence.storageClass" + value = "gp3" + } + + # SigNoz (alertmanager) storage configuration + set { + name = "alertmanager.persistence.storageClass" + value = "gp3" + } + + # ClickHouse resource configuration for loadtest + # Default 100m CPU and 200Mi memory are way too low for high-volume telemetry + set { + name = "clickhouse.resources.requests.cpu" + value = "2000m" + } + + set { + name = "clickhouse.resources.requests.memory" + value = "4Gi" + } + + set { + name = "clickhouse.resources.limits.cpu" + value = "4000m" + } + + set { + name = "clickhouse.resources.limits.memory" + value = "8Gi" + } + + # OTEL Collector resource configuration for loadtest + set { + name = "otelCollector.resources.requests.memory" + value = "8Gi" + } + + set { + name = "otelCollector.resources.limits.memory" + value = "12Gi" + } + + set { + name = "otelCollector.resources.requests.cpu" + value = "1000m" + } + + set { + name = "otelCollector.resources.limits.cpu" + value = "4000m" + } + + # Only need 1 replica since we have 1 LoadBalancer endpoint + set { + name = "otelCollector.replicaCount" + value = "1" + } + + # Uncomment to force redeployment + # + # set { + # name = "redeployAt" + # value = timestamp() + # } + + depends_on = [ + module.eks, + kubernetes_storage_class_v1.gp3, + helm_release.aws_load_balancer_controller + ] +} + +resource "kubernetes_secret" "otel_bearer_token" { + metadata { + name = "signoz-otel-bearer-token" + namespace = "signoz" + } + + type = "Opaque" + + data = { + OTEL_BEARER_TOKEN = var.otel_bearer_token + } +} + +resource "null_resource" "patch_otel_collector_bearer_env" { + triggers = { + token_hash = sha256(var.otel_bearer_token) + helm_release = tostring(helm_release.signoz.metadata[0].revision) + } + + provisioner "local-exec" { + command = <<-EOT + set -euo pipefail + kubectl -n signoz set env deployment/signoz-otel-collector \ + --containers=collector \ + --from=secret/${kubernetes_secret.otel_bearer_token.metadata[0].name} + EOT + } + + depends_on = [ + helm_release.signoz, + kubernetes_secret.otel_bearer_token + ] +} + +data "aws_lb" "signoz" { + name = local.signoz_alb_name + depends_on = [null_resource.wait_for_signoz_alb] +} + + +# Route53 records may require a 2nd apply if the load balancer changes in the EKS cluster. +resource "aws_route53_record" "signoz" { + zone_id = data.aws_route53_zone.dogfood.zone_id + name = local.signoz_domain + type = "A" + + alias { + name = data.aws_lb.signoz.dns_name + zone_id = data.aws_lb.signoz.zone_id + evaluate_target_health = true + } +} + +resource "null_resource" "wait_for_signoz_alb" { + depends_on = [helm_release.signoz] + + triggers = { + alb_name = local.signoz_alb_name + run_id = timestamp() + } + + provisioner "local-exec" { + command = <<-EOT + set -euo pipefail + for i in $(seq 1 60); do + if aws elbv2 describe-load-balancers --names "${local.signoz_alb_name}" --region "${data.aws_region.current.region}" >/dev/null 2>&1; then + exit 0 + fi + sleep 10 + done + echo "Timed out waiting for ALB ${local.signoz_alb_name}" >&2 + exit 1 + EOT + } +} + +resource "aws_route53_record" "otlp" { + zone_id = data.aws_route53_zone.dogfood.zone_id + name = local.otlp_domain + type = "A" + + alias { + name = data.aws_lb.signoz.dns_name + zone_id = data.aws_lb.signoz.zone_id + evaluate_target_health = true + } +} diff --git a/infrastructure/dogfood/terraform/aws-tf-module/signoz/otel-collector-values.yaml b/infrastructure/dogfood/terraform/aws-tf-module/signoz/otel-collector-values.yaml new file mode 100644 index 0000000000..3c2181d048 --- /dev/null +++ b/infrastructure/dogfood/terraform/aws-tf-module/signoz/otel-collector-values.yaml @@ -0,0 +1,84 @@ +# OTEL Collector configuration overrides for production stability +# These settings fix ClickHouse timeout issues by adding proper timeouts, +# retry logic, and sending queues. + +otelCollector: + # Enable low cardinality exception grouping to reduce exception noise + lowCardinalityExceptionGrouping: true + + config: + extensions: + health_check: + endpoint: 0.0.0.0:13133 + bearertokenauth: + scheme: "Bearer" + token: ${env:OTEL_BEARER_TOKEN} + + receivers: + otlp: + protocols: + grpc: + auth: + authenticator: bearertokenauth + http: + auth: + authenticator: bearertokenauth + + exporters: + # Logs exporter with balanced queue sizes for throughput and memory + clickhouselogsexporter: + timeout: 60s + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 300s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 10000 + + # Traces exporter must handle spans after tail_sampling + clickhousetraces: + timeout: 120s + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 300s + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 10000 + + # Metadata exporter with increased timeout and retry support + metadataexporter: + timeout: 60s + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 300s + + processors: + # Optimized batch processor to send data more frequently + # Smaller batches reduce memory pressure + batch: + send_batch_size: 2000 + timeout: 5s + + # Memory limiter to prevent out-of-memory by applying back pressure + memory_limiter: + check_interval: 1s + limit_mib: 11264 + spike_limit_mib: 512 + + service: + extensions: + - health_check + - bearertokenauth + pipelines: + traces: + processors: + - memory_limiter + - batch diff --git a/infrastructure/dogfood/terraform/aws-tf-module/signoz/outputs.tf b/infrastructure/dogfood/terraform/aws-tf-module/signoz/outputs.tf new file mode 100644 index 0000000000..7330fe41b0 --- /dev/null +++ b/infrastructure/dogfood/terraform/aws-tf-module/signoz/outputs.tf @@ -0,0 +1,28 @@ +output "cluster_name" { + description = "EKS cluster name" + value = module.eks.cluster_name +} + +output "cluster_endpoint" { + description = "EKS cluster endpoint" + value = module.eks.cluster_endpoint +} + +output "cluster_certificate_authority_data" { + description = "EKS cluster CA certificate" + value = module.eks.cluster_certificate_authority_data +} + +output "configure_kubectl" { + value = "aws eks update-kubeconfig --region ${data.aws_region.current.region} --name ${module.eks.cluster_name}" +} + +output "signoz_ui_url" { + value = "https://${local.signoz_domain}" +} + +# Output for programmatic access - internal LoadBalancer hostname +output "otel_collector_endpoint" { + description = "Internal OTLP collector endpoint (https://host:port)" + value = "https://${local.otlp_domain}:443" +} diff --git a/infrastructure/dogfood/terraform/aws-tf-module/signoz/scripts/send-test-telemetry.sh b/infrastructure/dogfood/terraform/aws-tf-module/signoz/scripts/send-test-telemetry.sh new file mode 100755 index 0000000000..3c9d6e996e --- /dev/null +++ b/infrastructure/dogfood/terraform/aws-tf-module/signoz/scripts/send-test-telemetry.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +NAMESPACE="${NAMESPACE:-signoz}" +ENDPOINT="${OTLP_ENDPOINT:-otlp.signoz.dogfood.fleetdm.com:443}" +TOKEN="${OTEL_BEARER_TOKEN:-}" +DURATION="${DURATION:-15s}" +IMAGE="${TELEMETRYGEN_IMAGE:-ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest}" + +if [[ -z "${TOKEN}" ]]; then + echo "OTEL_BEARER_TOKEN is required." >&2 + exit 1 +fi + +kubectl -n "${NAMESPACE}" run signoz-telemetrygen \ + --rm -i --restart=Never \ + --image "${IMAGE}" \ + --command -- /telemetrygen traces \ + --otlp-endpoint "${ENDPOINT}" \ + --otlp-header "authorization=\"Bearer ${TOKEN}\"" \ + --duration "${DURATION}"