Added terraform files for Signoz OTEL backend. (#34058)

**Related issue:** Resolves #32331 This PR allows us to run loadtest with SigNoz OTEL backend by adding `-var=enable_otel=true` SigNoz is deployed via Helm chart. Enhancements needed (in future PR): - put SigNoz UI behind VPN - combine the new eks-vpc with shared fleet-vpc - make SigNoz shared, so multiple loadtests use the same instance? (But what about updating to it to latest version?) Next steps: - Enable SigNoz in Dogfood environment - SigNoz by default [keeps 15 days of logs and traces](https://signoz.io/docs/userguide/retention-period), which is quite a bit. How much would that cost us and should we reduce it?  ## Summary by CodeRabbit - New Features - Optional OpenTelemetry tracing with SigNoz via a new enable_otel flag. - Conditional deployment of a SigNoz stack (managed EKS, storage, Helm-based apps) with internal OTLP collector endpoint. - New outputs to retrieve OTLP endpoint, cluster name, and a kubectl configuration command. - Documentation - Added guidance for deploying and using SigNoz with load testing. - Updated examples to include -var=enable_otel=true. - Chores - Introduced required providers to support Helm and Kubernetes resources.
2025-10-10 21:53:04 -05:00
parent 56e12f4aca
commit aef9b8400c
12 changed files with 500 additions and 22 deletions
@@ -0,0 +1,31 @@
+# EKS VPC for Fleet Loadtesting
+
+Dedicated VPC for EKS workloads (SigNoz) with proper Kubernetes tags.
+
+## Architecture
+
+- **CIDR**: 10.20.0.0/16
+- **Subnets**: 2 AZs (us-east-2a, us-east-2b)
+  - Private: 10.20.1.0/24, 10.20.2.0/24
+  - Public: 10.20.101.0/24, 10.20.102.0/24
+- **NAT**: Single NAT gateway (cost optimization)
+- **Tags**: Pre-configured for EKS/Kubernetes
+
+## Usage
+
+This VPC is deployed per workspace:
+
+```bash
+cd infrastructure/loadtesting/terraform/eks-vpc
+terraform workspace new <workspace_name>
+terraform apply
+```
+
+The VPC outputs are consumed by the SigNoz module via terraform remote state.
+
+## Why Separate VPC?
+
+- EKS requires specific subnet tags (`kubernetes.io/cluster/*`)
+- Can't modify shared fleet-vpc tags (different terraform state)
+- Avoids VPC limit issues (dedicated EKS VPC)
+- Clean separation of concerns
@@ -0,0 +1,66 @@
+terraform {
+  required_version = ">= 1.5"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.68.0"
+    }
+  }
+
+  backend "s3" {
+    bucket         = "fleet-terraform-state20220408141538466600000002"
+    key            = "loadtesting/shared/eks-vpc/terraform.tfstate"
+    region         = "us-east-2"
+    encrypt        = true
+    kms_key_id     = "9f98a443-ffd7-4dbe-a9c3-37df89b2e42a"
+    dynamodb_table = "tf-remote-state-lock"
+    assume_role = {
+      role_arn = "arn:aws:iam::353365949058:role/terraform-loadtesting"
+    }
+  }
+}
+
+provider "aws" {
+  region = "us-east-2"
+  default_tags {
+    tags = {
+      environment = terraform.workspace
+      terraform   = "https://github.com/fleetdm/fleet/tree/main/infrastructure/loadtesting/terraform/eks-vpc"
+      state       = "s3://fleet-terraform-state20220408141538466600000002/loadtesting/${terraform.workspace}/loadtesting/eks-vpc/terraform.tfstate"
+    }
+  }
+}
+
+# Shared VPC for EKS workloads with proper Kubernetes tags
+# This VPC is shared across all workspaces (like fleet-vpc)
+module "vpc" {
+  source  = "terraform-aws-modules/vpc/aws"
+  version = "~> 5.0"
+
+  name = "eks-vpc"
+  cidr = "10.20.0.0/16"
+
+  azs             = ["us-east-2a", "us-east-2b"]
+  private_subnets = ["10.20.1.0/24", "10.20.2.0/24"]
+  public_subnets  = ["10.20.101.0/24", "10.20.102.0/24"]
+
+  enable_nat_gateway   = true
+  single_nat_gateway   = true
+  enable_dns_hostnames = true
+
+  # Tags required for EKS - role tags are required on subnets
+  public_subnet_tags = {
+    "kubernetes.io/role/elb" = 1
+  }
+
+  private_subnet_tags = {
+    "kubernetes.io/role/internal-elb" = 1
+  }
+
+  # Note: Kubernetes cluster-specific tags are added by the signoz module
+  # when creating each EKS cluster, not at the VPC level
+  tags = {
+    "shared" = "true"
+  }
+}
@@ -0,0 +1,11 @@
+output "vpc" {
+  description = "VPC module outputs for EKS"
+  value = {
+    vpc_id             = module.vpc.vpc_id
+    private_subnets    = module.vpc.private_subnets
+    public_subnets     = module.vpc.public_subnets
+    vpc_cidr_block     = module.vpc.vpc_cidr_block
+    nat_gateway_ids    = module.vpc.natgw_ids
+    azs                = module.vpc.azs
+  }
+}
@@ -67,9 +67,27 @@ Additionally, refer to the [Reference Architecture sizing recommendations](https
    Below is an example with all available variables.

    ```sh
-    terraform apply -var=tag=v4.72.0 -var=fleet_task_count=20 -var=fleet_task_memory=4096 -var=fleet_task_cpu=512 -var=database_instance_size=db.t4g.large -var=database_instance_count=3 -var=redis_instance_size=cache.t4g.small -var=redis_instance_count=3
+    terraform apply -var=tag=v4.72.0 -var=fleet_task_count=20 -var=fleet_task_memory=4096 -var=fleet_task_cpu=512 -var=database_instance_size=db.t4g.large -var=database_instance_count=3 -var=redis_instance_size=cache.t4g.small -var=redis_instance_count=3 -var=enable_otel=true
    ```

+## OpenTelemetry tracing with SigNoz
+
+By default, the loadtest environment uses Elastic APM. You can optionally use OpenTelemetry with SigNoz instead by setting `enable_otel=true`:
+
+```sh
+terraform apply -var=tag=v4.72.0 -var=enable_otel=true
+```
+
+This deploys both Fleet and SigNoz in a single command. See [../signoz/README.md](../signoz/README.md) for architecture details.
+
+### Accessing the SigNoz UI
+
+After deploying with `enable_otel=true`, get the SigNoz UI URL:
+
+```sh
+$(terraform output -raw signoz_configure_kubectl) && kubectl get svc signoz -n signoz -o jsonpath='http://{.status.loadBalancer.ingress[0].hostname}:8080'
+```
+
 # Destroy environment manually

 1. Clone the repository (if not already cloned)
@@ -2,33 +2,47 @@ locals {
  customer    = "fleet-${terraform.workspace}"
  prefix      = "fleet-${terraform.workspace}"
  fleet_image = "${aws_ecr_repository.fleet.repository_url}:${var.tag}-${split(":", data.docker_registry_image.dockerhub.sha256_digest)[1]}"
-  extra_environment_variables = {
-    CLOUDWATCH_NAMESPACE = "fleet-loadtest-migration"
-    CLOUDWATCH_REGION    = "us-east-2"
-    # PROMETHEUS_SCRAPE_URL = "http://localhost:8080/metrics"
+  # Tracing configuration - either OTEL or Elastic APM
+  otel_environment_variables = var.enable_otel ? {
+    OTEL_SERVICE_NAME               = terraform.workspace
+    OTEL_EXPORTER_OTLP_ENDPOINT     = "http://${module.signoz[0].otel_collector_endpoint}"
+    FLEET_LOGGING_TRACING_ENABLED   = "true"
+    FLEET_LOGGING_TRACING_TYPE      = "opentelemetry"
+  } : {}

+  elastic_apm_environment_variables = var.enable_otel ? {} : {
    ELASTIC_APM_SERVER_URL              = "https://loadtest.fleetdm.com:8200"
    ELASTIC_APM_SERVICE_NAME            = "fleet"
    ELASTIC_APM_ENVIRONMENT             = "${terraform.workspace}"
    ELASTIC_APM_TRANSACTION_SAMPLE_RATE = "0.004"
    ELASTIC_APM_SERVICE_VERSION         = "${var.tag}-${split(":", data.docker_registry_image.dockerhub.sha256_digest)[1]}"
-
-    FLEET_VULNERABILITIES_DATABASES_PATH           = "/home/fleet"
-    FLEET_OSQUERY_ENABLE_ASYNC_HOST_PROCESSING     = "false"
-    FLEET_LOGGING_JSON                             = "true"
-    FLEET_LOGGING_DEBUG                            = "true"
-    FLEET_LOGGING_TRACING_ENABLED                  = "true"
-    FLEET_LOGGING_TRACING_TYPE                     = "elasticapm"
-    FLEET_MYSQL_MAX_OPEN_CONNS                     = "10"
-    FLEET_MYSQL_READ_REPLICA_MAX_OPEN_CONNS        = "10"
-    FLEET_OSQUERY_ASYNC_HOST_REDIS_SCAN_KEYS_COUNT = "10000"
-    FLEET_REDIS_MAX_OPEN_CONNS                     = "500"
-    FLEET_REDIS_MAX_IDLE_CONNS                     = "500"
-
-    # Load TLS Certificate for RDS Authentication
-    FLEET_MYSQL_TLS_CA              = local.cert_path
-    FLEET_MYSQL_READ_REPLICA_TLS_CA = local.cert_path
+    FLEET_LOGGING_TRACING_ENABLED       = "true"
+    FLEET_LOGGING_TRACING_TYPE          = "elasticapm"
  }
+
+  extra_environment_variables = merge(
+    {
+      CLOUDWATCH_NAMESPACE = "fleet-loadtest-migration"
+      CLOUDWATCH_REGION    = "us-east-2"
+      # PROMETHEUS_SCRAPE_URL = "http://localhost:8080/metrics"
+
+      FLEET_VULNERABILITIES_DATABASES_PATH           = "/home/fleet"
+      FLEET_OSQUERY_ENABLE_ASYNC_HOST_PROCESSING     = "false"
+      FLEET_LOGGING_JSON                             = "true"
+      FLEET_LOGGING_DEBUG                            = "true"
+      FLEET_MYSQL_MAX_OPEN_CONNS                     = "10"
+      FLEET_MYSQL_READ_REPLICA_MAX_OPEN_CONNS        = "10"
+      FLEET_OSQUERY_ASYNC_HOST_REDIS_SCAN_KEYS_COUNT = "10000"
+      FLEET_REDIS_MAX_OPEN_CONNS                     = "500"
+      FLEET_REDIS_MAX_IDLE_CONNS                     = "500"
+
+      # Load TLS Certificate for RDS Authentication
+      FLEET_MYSQL_TLS_CA              = local.cert_path
+      FLEET_MYSQL_READ_REPLICA_TLS_CA = local.cert_path
+    },
+    local.otel_environment_variables,
+    local.elastic_apm_environment_variables
+  )
  extra_secrets = {
    FLEET_LICENSE_KEY = data.aws_secretsmanager_secret.license.arn
  }
@@ -12,6 +12,14 @@ terraform {
      source  = "paultyng/git"
      version = "~> 0.1.0"
    }
+    helm = {
+      source  = "hashicorp/helm"
+      version = "~> 2.11"
+    }
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "~> 2.23"
+    }
  }
  backend "s3" {
    bucket               = "fleet-terraform-state20220408141538466600000002"
@@ -55,6 +63,22 @@ data "terraform_remote_state" "shared" {
  }
 }

+data "terraform_remote_state" "eks_vpc" {
+  count   = var.enable_otel ? 1 : 0
+  backend = "s3"
+  config = {
+    bucket         = "fleet-terraform-state20220408141538466600000002"
+    key            = "loadtesting/shared/eks-vpc/terraform.tfstate"
+    region         = "us-east-2"
+    encrypt        = true
+    kms_key_id     = "9f98a443-ffd7-4dbe-a9c3-37df89b2e42a"
+    dynamodb_table = "tf-remote-state-lock"
+    assume_role = {
+      role_arn = "arn:aws:iam::353365949058:role/terraform-loadtesting"
+    }
+  }
+}
+
 provider "docker" {
  # Configuration options
  registry_auth {
@@ -64,4 +88,35 @@ provider "docker" {
  }
 }

-provider "git" {}
+provider "git" {}
+
+# Data sources for SigNoz EKS cluster authentication
+data "aws_eks_cluster" "signoz" {
+  count = var.enable_otel ? 1 : 0
+  name  = module.signoz[0].cluster_name
+}
+
+data "aws_eks_cluster_auth" "signoz" {
+  count = var.enable_otel ? 1 : 0
+  name  = module.signoz[0].cluster_name
+}
+
+# Helm provider for SigNoz EKS cluster
+provider "helm" {
+  alias = "signoz"
+
+  kubernetes {
+    host                   = var.enable_otel ? data.aws_eks_cluster.signoz[0].endpoint : ""
+    cluster_ca_certificate = var.enable_otel ? base64decode(data.aws_eks_cluster.signoz[0].certificate_authority[0].data) : ""
+    token                  = var.enable_otel ? data.aws_eks_cluster_auth.signoz[0].token : ""
+  }
+}
+
+# Kubernetes provider for SigNoz EKS cluster
+provider "kubernetes" {
+  alias = "signoz"
+
+  host                   = var.enable_otel ? data.aws_eks_cluster.signoz[0].endpoint : ""
+  cluster_ca_certificate = var.enable_otel ? base64decode(data.aws_eks_cluster.signoz[0].certificate_authority[0].data) : ""
+  token                  = var.enable_otel ? data.aws_eks_cluster_auth.signoz[0].token : ""
+}
@@ -0,0 +1,36 @@
+# SigNoz deployment for OpenTelemetry tracing
+# Conditionally deployed when var.enable_otel = true
+
+module "signoz" {
+  count  = var.enable_otel ? 1 : 0
+  source = "../signoz"
+
+  aws_region   = data.aws_region.current.region
+  cluster_name = "signoz-${terraform.workspace}"
+
+  # Use dedicated EKS VPC with proper Kubernetes tags
+  vpc_id     = data.terraform_remote_state.eks_vpc[0].outputs.vpc.vpc_id
+  subnet_ids = data.terraform_remote_state.eks_vpc[0].outputs.vpc.private_subnets
+
+  providers = {
+    aws        = aws
+    helm       = helm.signoz
+    kubernetes = kubernetes.signoz
+  }
+}
+
+# Outputs from SigNoz module
+output "signoz_cluster_name" {
+  description = "SigNoz EKS cluster name"
+  value       = var.enable_otel ? module.signoz[0].cluster_name : null
+}
+
+output "signoz_otel_collector_endpoint" {
+  description = "Internal OTLP collector endpoint for Fleet"
+  value       = var.enable_otel ? module.signoz[0].otel_collector_endpoint : null
+}
+
+output "signoz_configure_kubectl" {
+  description = "Command to configure kubectl for SigNoz"
+  value       = var.enable_otel ? module.signoz[0].configure_kubectl : null
+}
@@ -58,4 +58,10 @@ variable "redis_instance_count" {
    condition     = var.redis_instance_count >= 3
    error_message = "var.redis_instance_count must be greater than or equal to 3."
  }
+}
+
+variable "enable_otel" {
+  description = "Enable OpenTelemetry tracing with SigNoz instead of Elastic APM"
+  type        = bool
+  default     = false
 }
@@ -0,0 +1,24 @@
+# SigNoz module for Fleet loadtest OTEL tracing
+
+OpenTelemetry observability backend module, deployed conditionally from the Fleet loadtest infrastructure.
+
+## Usage
+
+This module is deployed automatically when `enable_otel=true` is set in the parent loadtest infrastructure:
+
+```bash
+cd infrastructure/loadtesting/terraform/infra
+terraform workspace new <workspace_name>
+terraform apply -var=enable_otel=true
+```
+
+## What gets deployed
+
+- **EKS cluster** for SigNoz in shared eks-vpc (K8s 1.31, 2x t3.large nodes)
+- **OTLP endpoint**: Internal LoadBalancer (not publicly accessible)
+- **SigNoz UI**: Public LoadBalancer on port 8080
+- **Storage**: EBS CSI driver with gp2 default storage class
+
+## Architecture
+
+SigNoz uses the shared `eks-vpc` (not fleet-vpc) which has proper Kubernetes subnet tags. The eks-vpc is shared across all workspaces (like fleet-vpc), but each workspace deploys its own SigNoz EKS cluster within it.
@@ -0,0 +1,155 @@
+terraform {
+  required_version = ">= 1.5"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.68.0"
+    }
+    helm = {
+      source  = "hashicorp/helm"
+      version = "~> 2.11"
+      configuration_aliases = [helm]
+    }
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "~> 2.23"
+      configuration_aliases = [kubernetes]
+    }
+  }
+}
+
+locals {
+  cluster_name = var.cluster_name
+}
+
+# Use shared fleet VPC
+module "eks" {
+  source  = "terraform-aws-modules/eks/aws"
+  version = "~> 21.0"
+
+  name               = local.cluster_name
+  kubernetes_version = "1.31"
+
+  endpoint_public_access = true
+
+  vpc_id     = var.vpc_id
+  subnet_ids = var.subnet_ids
+
+  # IMPORTANT: Install critical addons BEFORE node group to avoid circular dependency
+  # Nodes need VPC CNI to become Ready, but terraform waits for nodes to be Ready
+  # before creating addons. This causes a deadlock where nodes are stuck NotReady.
+  # Solution: Use addons with before_compute=true to install VPC CNI before node group completes.
+  addons = {
+    vpc-cni = {
+      most_recent    = true
+      before_compute = true
+    }
+    kube-proxy = {
+      most_recent    = true
+      before_compute = true
+    }
+    coredns = {
+      most_recent    = true
+      before_compute = true
+    }
+  }
+
+  # Managed node group
+  eks_managed_node_groups = {
+    default = {
+      min_size       = 2
+      max_size       = 2
+      desired_size   = 2
+      instance_types = ["t3.large"]
+    }
+  }
+
+  # Enable cluster creator admin access
+  enable_cluster_creator_admin_permissions = true
+
+  # Enable OIDC provider for IRSA (IAM Roles for Service Accounts)
+  enable_irsa = true
+}
+
+# IAM Role for EBS CSI Driver Service Account (IRSA)
+module "ebs_csi_irsa_role" {
+  source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
+  version = "~> 5.0"
+
+  role_name             = "${local.cluster_name}-ebs-csi-driver"
+  attach_ebs_csi_policy = true
+
+  oidc_providers = {
+    main = {
+      provider_arn               = module.eks.oidc_provider_arn
+      namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"]
+    }
+  }
+}
+
+# EBS CSI Driver addon with IRSA support
+# This must be created separately to avoid circular dependency with OIDC provider
+resource "aws_eks_addon" "ebs_csi" {
+  cluster_name             = module.eks.cluster_name
+  addon_name               = "aws-ebs-csi-driver"
+  addon_version            = data.aws_eks_addon_version.ebs_csi.version
+  service_account_role_arn = module.ebs_csi_irsa_role.iam_role_arn
+
+  depends_on = [
+    module.ebs_csi_irsa_role,
+    module.eks
+  ]
+}
+
+data "aws_eks_addon_version" "ebs_csi" {
+  addon_name         = "aws-ebs-csi-driver"
+  kubernetes_version = module.eks.cluster_version
+  most_recent        = true
+}
+
+# SigNoz via Helm
+resource "helm_release" "signoz" {
+  name       = "signoz"
+  repository = "https://charts.signoz.io"
+  chart      = "signoz"
+  namespace  = "signoz"
+  timeout    = 900
+
+  create_namespace = true
+
+  set {
+    name  = "cloud"
+    value = "false"
+  }
+
+  set {
+    name  = "signoz.service.type"
+    value = "LoadBalancer"
+  }
+
+  # OTLP collector should be internal only (not publicly accessible)
+  set {
+    name  = "otelCollector.service.type"
+    value = "LoadBalancer"
+  }
+
+  set {
+    name  = "otelCollector.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-scheme"
+    value = "internal"
+  }
+
+  set {
+    name  = "clickhouse.persistence.size"
+    value = "20Gi"
+  }
+
+  set {
+    name  = "clickhouse.persistence.storageClassName"
+    value = "gp3"
+  }
+
+  depends_on = [
+    module.eks
+  ]
+}
@@ -0,0 +1,42 @@
+output "cluster_name" {
+  description = "EKS cluster name"
+  value       = module.eks.cluster_name
+}
+
+output "cluster_endpoint" {
+  description = "EKS cluster endpoint"
+  value       = module.eks.cluster_endpoint
+}
+
+output "cluster_certificate_authority_data" {
+  description = "EKS cluster CA certificate"
+  value       = module.eks.cluster_certificate_authority_data
+}
+
+output "configure_kubectl" {
+  value = "aws eks update-kubeconfig --region ${var.aws_region} --name ${module.eks.cluster_name}"
+}
+
+output "get_signoz_ui_url" {
+  value = "kubectl get svc -n signoz signoz -o jsonpath='{.status.loadBalancer.ingress[0].hostname}':8080"
+}
+
+output "get_otlp_endpoint" {
+  value = "kubectl get svc -n signoz signoz-otel-collector -o jsonpath='{.status.loadBalancer.ingress[0].hostname}':4317"
+}
+
+# Data source to get the OTLP collector service
+data "kubernetes_service" "otlp_collector" {
+  metadata {
+    name      = "signoz-otel-collector"
+    namespace = "signoz"
+  }
+
+  depends_on = [helm_release.signoz]
+}
+
+# Output for programmatic access - internal LoadBalancer hostname
+output "otel_collector_endpoint" {
+  description = "Internal OTLP collector endpoint (hostname:port)"
+  value       = try("${data.kubernetes_service.otlp_collector.status[0].load_balancer[0].ingress[0].hostname}:4317", "LoadBalancer not ready yet")
+}
@@ -0,0 +1,20 @@
+variable "aws_region" {
+  description = "AWS region"
+  type        = string
+  default     = "us-east-2"
+}
+
+variable "cluster_name" {
+  description = "Name of the EKS cluster for SigNoz"
+  type        = string
+}
+
+variable "vpc_id" {
+  description = "VPC ID for SigNoz EKS cluster (shared fleet VPC)"
+  type        = string
+}
+
+variable "subnet_ids" {
+  description = "Subnet IDs for SigNoz EKS cluster (private subnets from fleet VPC)"
+  type        = list(string)
+}