Dogfood signoz (#38569)
This commit is contained in:
Robert Fairburn
@@ -38,6 +38,7 @@ env:
|
||||
TF_VAR_elastic_token: ${{ secrets.ELASTIC_APM_SECRET_TOKEN }}
|
||||
TF_VAR_geolite2_license: ${{ secrets.MAXMIND_LICENSE }}
|
||||
TF_VAR_dogfood_sidecar_enroll_secret: ${{ secrets.DOGFOOD_SERVERS_CANARY_ENROLL_SECRET }}
|
||||
TF_VAR_otel_bearer_token: ${{ secrets.DOGFOOD_SIGNOZ_OTEL_BEARER_TOKEN }}
|
||||
TF_VAR_android_service_credentials: ${{ secrets.FLEET_DEV_ANDROID_SERVICE_CREDENTIALS }}
|
||||
TF_VAR_cloudfront_public_key: ${{ secrets.CLOUDFRONT_SIGNING_PUBLIC_KEY }}
|
||||
TF_VAR_cloudfront_private_key: ${{ secrets.CLOUDFRONT_SIGNING_PRIVATE_KEY }}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
name: Deploy Dogfood SigNoz
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
workspace:
|
||||
description: "Terraform workspace to deploy SigNoz into."
|
||||
required: true
|
||||
default: "fleet"
|
||||
dry_run:
|
||||
description: Dry run only? No "terraform apply"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id}}
|
||||
cancel-in-progress: true
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
working-directory: infrastructure/dogfood/terraform/aws-tf-module/signoz
|
||||
|
||||
env:
|
||||
AWS_REGION: us-east-2
|
||||
AWS_IAM_ROLE: arn:aws:iam::160035666661:role/github-actions-role
|
||||
TF_ACTIONS_WORKING_DIR: infrastructure/dogfood/terraform/aws-tf-module/signoz
|
||||
TF_WORKSPACE: ${{ github.event.inputs.workspace }}
|
||||
TF_VAR_otel_bearer_token: ${{ secrets.DOGFOOD_SIGNOZ_OTEL_BEARER_TOKEN }}
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
name: Deploy Dogfood SigNoz
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
|
||||
- id: fail-on-main
|
||||
run: "false"
|
||||
if: ${{ github.ref == 'main' }}
|
||||
|
||||
- uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 # v1.7.0
|
||||
with:
|
||||
role-to-assume: ${{ env.AWS_IAM_ROLE }}
|
||||
aws-region: ${{ env.AWS_REGION }}
|
||||
|
||||
- uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3
|
||||
with:
|
||||
terraform_version: 1.10.2
|
||||
terraform_wrapper: false
|
||||
|
||||
- name: Set up kubectl
|
||||
uses: azure/setup-kubectl@18a212d6b26e63d2e08d083e82f0597a6095f08c # v4.0.0
|
||||
with:
|
||||
version: v1.31.0
|
||||
|
||||
- name: Configure kubeconfig
|
||||
run: aws eks update-kubeconfig --region "${AWS_REGION}" --name signoz
|
||||
|
||||
- name: Terraform Init
|
||||
id: init
|
||||
run: terraform init
|
||||
- name: Terraform fmt
|
||||
id: fmt
|
||||
run: terraform fmt -check
|
||||
continue-on-error: true
|
||||
- name: Terraform Validate
|
||||
id: validate
|
||||
run: terraform validate -no-color
|
||||
- name: Terraform Plan
|
||||
id: plan
|
||||
run: terraform plan -no-color
|
||||
continue-on-error: true
|
||||
- name: Terraform Apply
|
||||
if: inputs.dry_run == false
|
||||
id: apply
|
||||
run: terraform apply -auto-approve
|
||||
@@ -70,6 +70,7 @@ data "aws_iam_policy_document" "gha-permissions" {
|
||||
"s3:*",
|
||||
"lambda:*",
|
||||
"ecs:*",
|
||||
"eks:*",
|
||||
"rds:*",
|
||||
"rds-data:*",
|
||||
"secretsmanager:*",
|
||||
|
||||
@@ -47,6 +47,10 @@ variable "cloudfront_private_key" {}
|
||||
variable "webhook_url" {
|
||||
description = "Webhook URL used for Webhook Logging Destination"
|
||||
}
|
||||
variable "otel_bearer_token" {
|
||||
sensitive = true
|
||||
description = "Bearer token for OTLP ingest into SigNoz."
|
||||
}
|
||||
|
||||
data "aws_caller_identity" "current" {}
|
||||
|
||||
@@ -55,9 +59,13 @@ locals {
|
||||
fleet_image = var.fleet_image # Set this to the version of fleet to be deployed
|
||||
geolite2_image = "${aws_ecr_repository.fleet.repository_url}:${split(":", var.fleet_image)[1]}-geolite2-${formatdate("YYYYMMDDhhmm", timestamp())}"
|
||||
extra_environment_variables = {
|
||||
FLEET_LICENSE_KEY = var.fleet_license
|
||||
FLEET_LOGGING_DEBUG = "true"
|
||||
FLEET_LOGGING_JSON = "true"
|
||||
FLEET_LICENSE_KEY = var.fleet_license
|
||||
FLEET_LOGGING_DEBUG = "true"
|
||||
FLEET_LOGGING_JSON = "true"
|
||||
FLEET_LOGGING_TRACING_ENABLED = "true"
|
||||
FLEET_LOGGING_TRACING_TYPE = "opentelemetry"
|
||||
OTEL_SERVICE_NAME = local.customer
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT = "https://otlp.signoz.dogfood.fleetdm.com"
|
||||
# FLEET_LOGGING_TRACING_ENABLED = "true"
|
||||
# FLEET_LOGGING_TRACING_TYPE = "elasticapm"
|
||||
FLEET_MYSQL_MAX_OPEN_CONNS = "10"
|
||||
@@ -85,6 +93,9 @@ locals {
|
||||
sentry_secrets = {
|
||||
FLEET_SENTRY_DSN = "${aws_secretsmanager_secret.sentry.arn}:FLEET_SENTRY_DSN::"
|
||||
}
|
||||
signoz_secrets = {
|
||||
OTEL_EXPORTER_OTLP_HEADERS = "${aws_secretsmanager_secret.signoz_otel_bearer_token.arn}:OTEL_EXPORTER_OTLP_HEADERS::"
|
||||
}
|
||||
# idp_metadata_file = "${path.module}/files/idp-metadata.xml"
|
||||
|
||||
/*
|
||||
@@ -201,13 +212,19 @@ module "main" {
|
||||
)
|
||||
extra_execution_iam_policies = concat(
|
||||
module.mdm.extra_execution_iam_policies,
|
||||
[aws_iam_policy.sentry.arn, aws_iam_policy.osquery_sidecar.arn, aws_iam_policy.entra_conditional_access.arn],
|
||||
[
|
||||
aws_iam_policy.sentry.arn,
|
||||
aws_iam_policy.osquery_sidecar.arn,
|
||||
aws_iam_policy.entra_conditional_access.arn,
|
||||
aws_iam_policy.signoz_otel_bearer_token.arn
|
||||
],
|
||||
module.cloudfront-software-installers.extra_execution_iam_policies,
|
||||
) #, module.saml_auth_proxy.fleet_extra_execution_policies)
|
||||
extra_secrets = merge(
|
||||
local.entra_conditional_access_secrets,
|
||||
module.mdm.extra_secrets,
|
||||
local.sentry_secrets,
|
||||
local.signoz_secrets,
|
||||
module.cloudfront-software-installers.extra_secrets
|
||||
)
|
||||
private_key_secret_name = "${local.customer}-fleet-server-private-key"
|
||||
@@ -427,6 +444,31 @@ data "aws_iam_policy_document" "sentry" {
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret" "signoz_otel_bearer_token" {
|
||||
name = "${local.customer}-signoz-otel-bearer-token"
|
||||
}
|
||||
|
||||
resource "aws_secretsmanager_secret_version" "signoz_otel_bearer_token" {
|
||||
secret_id = aws_secretsmanager_secret.signoz_otel_bearer_token.id
|
||||
secret_string = jsonencode({
|
||||
OTEL_EXPORTER_OTLP_HEADERS = "Authorization=Bearer ${var.otel_bearer_token}"
|
||||
})
|
||||
}
|
||||
|
||||
resource "aws_iam_policy" "signoz_otel_bearer_token" {
|
||||
name = "fleet-signoz-otel-bearer-token-policy"
|
||||
policy = data.aws_iam_policy_document.signoz_otel_bearer_token.json
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "signoz_otel_bearer_token" {
|
||||
statement {
|
||||
actions = [
|
||||
"secretsmanager:GetSecretValue",
|
||||
]
|
||||
resources = [aws_secretsmanager_secret.signoz_otel_bearer_token.arn]
|
||||
}
|
||||
}
|
||||
|
||||
module "migrations" {
|
||||
depends_on = [
|
||||
module.geolite2
|
||||
@@ -761,3 +803,7 @@ module "cloudfront-software-installers" {
|
||||
enable_logging = true
|
||||
logging_s3_bucket = module.logging_alb.log_s3_bucket_id
|
||||
}
|
||||
|
||||
output "vpc" {
|
||||
value = module.main.vpc
|
||||
}
|
||||
|
||||
@@ -0,0 +1,569 @@
|
||||
terraform {
|
||||
required_version = ">= 1.5"
|
||||
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = ">= 5.68.0"
|
||||
}
|
||||
helm = {
|
||||
source = "hashicorp/helm"
|
||||
version = "~> 2.11"
|
||||
}
|
||||
kubernetes = {
|
||||
source = "hashicorp/kubernetes"
|
||||
version = "~> 2.23"
|
||||
}
|
||||
}
|
||||
|
||||
backend "s3" {
|
||||
bucket = "fleet-terraform-state20220408141538466600000002"
|
||||
key = "dogfood/signoz/terraform.tfstate"
|
||||
workspace_key_prefix = "dogfood"
|
||||
region = "us-east-2"
|
||||
encrypt = true
|
||||
kms_key_id = "9f98a443-ffd7-4dbe-a9c3-37df89b2e42a"
|
||||
dynamodb_table = "tf-remote-state-lock"
|
||||
assume_role = {
|
||||
role_arn = "arn:aws:iam::353365949058:role/terraform-dogfood"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
default_tags {
|
||||
tags = {
|
||||
environment = "dogfood-signoz"
|
||||
terraform = "https://github.com/fleetdm/fleet/tree/main/infrastructure/dogfood/terraform/aws-tf-module/signoz"
|
||||
workspace = terraform.workspace
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Read shared VPC from remote state
|
||||
data "terraform_remote_state" "dogfood" {
|
||||
backend = "s3"
|
||||
workspace = "fleet"
|
||||
config = {
|
||||
bucket = "fleet-terraform-remote-state"
|
||||
key = "fleet"
|
||||
region = "us-east-2"
|
||||
dynamodb_table = "fleet-terraform-state-lock"
|
||||
}
|
||||
}
|
||||
|
||||
# Providers for Helm/Kubernetes after cluster is created
|
||||
provider "helm" {
|
||||
kubernetes {
|
||||
host = module.eks.cluster_endpoint
|
||||
cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
|
||||
token = data.aws_eks_cluster_auth.signoz.token
|
||||
}
|
||||
}
|
||||
|
||||
provider "kubernetes" {
|
||||
host = module.eks.cluster_endpoint
|
||||
cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
|
||||
token = data.aws_eks_cluster_auth.signoz.token
|
||||
}
|
||||
|
||||
data "aws_region" "current" {}
|
||||
|
||||
data "aws_eks_cluster_auth" "signoz" {
|
||||
name = module.eks.cluster_name
|
||||
}
|
||||
|
||||
data "aws_route53_zone" "dogfood" {
|
||||
name = "dogfood.fleetdm.com."
|
||||
}
|
||||
|
||||
locals {
|
||||
cluster_name = "signoz"
|
||||
signoz_domain = "signoz.dogfood.fleetdm.com"
|
||||
otlp_domain = "otlp.signoz.dogfood.fleetdm.com"
|
||||
signoz_alb_name = "signoz-dogfood"
|
||||
alb_subnets = join(",", compact(data.terraform_remote_state.dogfood.outputs.vpc.private_subnets))
|
||||
common_ingress_annotations = {
|
||||
"alb.ingress.kubernetes.io/scheme" = "internal"
|
||||
"alb.ingress.kubernetes.io/subnets" = local.alb_subnets
|
||||
"alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTPS\":443}]"
|
||||
"alb.ingress.kubernetes.io/certificate-arn" = module.acm.acm_certificate_arn
|
||||
"alb.ingress.kubernetes.io/backend-protocol" = "HTTP"
|
||||
"alb.ingress.kubernetes.io/target-type" = "ip"
|
||||
"alb.ingress.kubernetes.io/group.name" = local.signoz_alb_name
|
||||
"alb.ingress.kubernetes.io/load-balancer-name" = local.signoz_alb_name
|
||||
}
|
||||
signoz_ingress_annotations = local.common_ingress_annotations
|
||||
otlp_ingress_annotations = merge(local.common_ingress_annotations, {
|
||||
"alb.ingress.kubernetes.io/backend-protocol-version" = "GRPC"
|
||||
})
|
||||
}
|
||||
|
||||
module "acm" {
|
||||
source = "terraform-aws-modules/acm/aws"
|
||||
version = "4.3.1"
|
||||
|
||||
domain_name = local.signoz_domain
|
||||
subject_alternative_names = [local.otlp_domain]
|
||||
zone_id = data.aws_route53_zone.dogfood.zone_id
|
||||
|
||||
wait_for_validation = true
|
||||
}
|
||||
|
||||
# Use shared fleet VPC
|
||||
module "eks" {
|
||||
source = "terraform-aws-modules/eks/aws"
|
||||
version = "~> 21.0"
|
||||
|
||||
name = local.cluster_name
|
||||
kubernetes_version = "1.31"
|
||||
|
||||
endpoint_public_access = true
|
||||
|
||||
vpc_id = data.terraform_remote_state.dogfood.outputs.vpc.vpc_id
|
||||
subnet_ids = data.terraform_remote_state.dogfood.outputs.vpc.private_subnets
|
||||
|
||||
# IMPORTANT: Install critical addons BEFORE node group to avoid circular dependency
|
||||
# Nodes need VPC CNI to become Ready, but terraform waits for nodes to be Ready
|
||||
# before creating addons. This causes a deadlock where nodes are stuck NotReady.
|
||||
# Solution: Use addons with before_compute=true to install VPC CNI before node group completes.
|
||||
addons = {
|
||||
vpc-cni = {
|
||||
most_recent = true
|
||||
before_compute = true
|
||||
}
|
||||
kube-proxy = {
|
||||
most_recent = true
|
||||
before_compute = true
|
||||
}
|
||||
coredns = {
|
||||
most_recent = true
|
||||
before_compute = true
|
||||
}
|
||||
}
|
||||
|
||||
# Managed node group
|
||||
eks_managed_node_groups = {
|
||||
default = {
|
||||
min_size = 2
|
||||
max_size = 3
|
||||
desired_size = 2
|
||||
instance_types = ["t3.xlarge"]
|
||||
}
|
||||
}
|
||||
|
||||
# Enable cluster creator admin access
|
||||
enable_cluster_creator_admin_permissions = true
|
||||
|
||||
# Enable OIDC provider for IRSA (IAM Roles for Service Accounts)
|
||||
enable_irsa = true
|
||||
}
|
||||
|
||||
# IAM Role for EBS CSI Driver Service Account (IRSA)
|
||||
module "ebs_csi_irsa_role" {
|
||||
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
|
||||
version = "~> 5.0"
|
||||
|
||||
role_name = "${local.cluster_name}-ebs-csi-driver"
|
||||
attach_ebs_csi_policy = true
|
||||
|
||||
oidc_providers = {
|
||||
main = {
|
||||
provider_arn = module.eks.oidc_provider_arn
|
||||
namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# IAM Role for AWS Load Balancer Controller Service Account (IRSA)
|
||||
module "load_balancer_controller_irsa_role" {
|
||||
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
|
||||
version = "~> 5.0"
|
||||
|
||||
role_name = "${local.cluster_name}-aws-load-balancer-controller"
|
||||
|
||||
attach_load_balancer_controller_policy = true
|
||||
|
||||
oidc_providers = {
|
||||
main = {
|
||||
provider_arn = module.eks.oidc_provider_arn
|
||||
namespace_service_accounts = ["kube-system:aws-load-balancer-controller"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# EBS CSI Driver addon with IRSA support
|
||||
# This must be created separately to avoid circular dependency with OIDC provider
|
||||
resource "aws_eks_addon" "ebs_csi" {
|
||||
cluster_name = module.eks.cluster_name
|
||||
addon_name = "aws-ebs-csi-driver"
|
||||
addon_version = data.aws_eks_addon_version.ebs_csi.version
|
||||
service_account_role_arn = module.ebs_csi_irsa_role.iam_role_arn
|
||||
|
||||
depends_on = [
|
||||
module.ebs_csi_irsa_role,
|
||||
module.eks
|
||||
]
|
||||
}
|
||||
|
||||
data "aws_eks_addon_version" "ebs_csi" {
|
||||
addon_name = "aws-ebs-csi-driver"
|
||||
kubernetes_version = module.eks.cluster_version
|
||||
most_recent = true
|
||||
}
|
||||
|
||||
# Wait for EBS CSI driver to be active
|
||||
resource "time_sleep" "wait_for_ebs_csi" {
|
||||
depends_on = [aws_eks_addon.ebs_csi]
|
||||
|
||||
create_duration = "60s"
|
||||
}
|
||||
|
||||
# Create gp3 storage class using EBS CSI driver
|
||||
resource "kubernetes_storage_class_v1" "gp3" {
|
||||
metadata {
|
||||
name = "gp3"
|
||||
annotations = {
|
||||
"storageclass.kubernetes.io/is-default-class" = "true"
|
||||
}
|
||||
}
|
||||
|
||||
storage_provisioner = "ebs.csi.aws.com"
|
||||
reclaim_policy = "Delete"
|
||||
allow_volume_expansion = true
|
||||
volume_binding_mode = "WaitForFirstConsumer"
|
||||
|
||||
parameters = {
|
||||
type = "gp3"
|
||||
encrypted = "true"
|
||||
fsType = "ext4"
|
||||
}
|
||||
|
||||
depends_on = [time_sleep.wait_for_ebs_csi]
|
||||
}
|
||||
|
||||
# AWS Load Balancer Controller via Helm (managed add-on not available for 1.31 in all regions)
|
||||
resource "helm_release" "aws_load_balancer_controller" {
|
||||
name = "aws-load-balancer-controller"
|
||||
repository = "https://aws.github.io/eks-charts"
|
||||
chart = "aws-load-balancer-controller"
|
||||
namespace = "kube-system"
|
||||
|
||||
wait = true
|
||||
wait_for_jobs = false
|
||||
|
||||
set {
|
||||
name = "clusterName"
|
||||
value = module.eks.cluster_name
|
||||
}
|
||||
|
||||
set {
|
||||
name = "region"
|
||||
value = data.aws_region.current.region
|
||||
}
|
||||
|
||||
set {
|
||||
name = "vpcId"
|
||||
value = data.terraform_remote_state.dogfood.outputs.vpc.vpc_id
|
||||
}
|
||||
|
||||
set {
|
||||
name = "serviceAccount.create"
|
||||
value = "true"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "serviceAccount.name"
|
||||
value = "aws-load-balancer-controller"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
|
||||
value = module.load_balancer_controller_irsa_role.iam_role_arn
|
||||
}
|
||||
|
||||
depends_on = [
|
||||
module.eks,
|
||||
module.load_balancer_controller_irsa_role
|
||||
]
|
||||
}
|
||||
|
||||
# SigNoz via Helm
|
||||
resource "helm_release" "signoz" {
|
||||
name = "signoz"
|
||||
repository = "https://charts.signoz.io"
|
||||
chart = "signoz"
|
||||
namespace = "signoz"
|
||||
timeout = 3600 # 60 minutes - allows for worst-case AWS delays during both apply and destroy
|
||||
|
||||
create_namespace = true
|
||||
wait = true
|
||||
wait_for_jobs = false
|
||||
|
||||
# OTEL Collector configuration overrides for production stability
|
||||
values = [
|
||||
file("${path.module}/otel-collector-values.yaml"),
|
||||
yamlencode({
|
||||
signoz = {
|
||||
ingress = {
|
||||
annotations = local.signoz_ingress_annotations
|
||||
}
|
||||
}
|
||||
otelCollector = {
|
||||
ingress = {
|
||||
annotations = local.otlp_ingress_annotations
|
||||
}
|
||||
}
|
||||
})
|
||||
]
|
||||
|
||||
set {
|
||||
name = "cloud"
|
||||
value = "false"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "signoz.service.type"
|
||||
value = "ClusterIP"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "signoz.ingress.enabled"
|
||||
value = "true"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "signoz.ingress.className"
|
||||
value = "alb"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "signoz.ingress.hosts[0].host"
|
||||
value = local.signoz_domain
|
||||
}
|
||||
|
||||
set {
|
||||
name = "signoz.ingress.hosts[0].paths[0].path"
|
||||
value = "/*"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "signoz.ingress.hosts[0].paths[0].pathType"
|
||||
value = "ImplementationSpecific"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "signoz.ingress.hosts[0].paths[0].port"
|
||||
value = "8080"
|
||||
}
|
||||
|
||||
# OTLP collector should be internal only (not publicly accessible)
|
||||
set {
|
||||
name = "otelCollector.service.type"
|
||||
value = "ClusterIP"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "otelCollector.ingress.enabled"
|
||||
value = "true"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "otelCollector.ingress.className"
|
||||
value = "alb"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "otelCollector.ingress.hosts[0].host"
|
||||
value = local.otlp_domain
|
||||
}
|
||||
|
||||
set {
|
||||
name = "otelCollector.ingress.hosts[0].paths[0].path"
|
||||
value = "/*"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "otelCollector.ingress.hosts[0].paths[0].pathType"
|
||||
value = "ImplementationSpecific"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "otelCollector.ingress.hosts[0].paths[0].port"
|
||||
value = "4317"
|
||||
}
|
||||
|
||||
|
||||
# Clickhouse storage configuration
|
||||
set {
|
||||
name = "clickhouse.persistence.size"
|
||||
value = "200Gi"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "clickhouse.persistence.storageClass"
|
||||
value = "gp3"
|
||||
}
|
||||
|
||||
# Zookeeper storage configuration
|
||||
set {
|
||||
name = "zookeeper.persistence.storageClass"
|
||||
value = "gp3"
|
||||
}
|
||||
|
||||
# SigNoz (alertmanager) storage configuration
|
||||
set {
|
||||
name = "alertmanager.persistence.storageClass"
|
||||
value = "gp3"
|
||||
}
|
||||
|
||||
# ClickHouse resource configuration for loadtest
|
||||
# Default 100m CPU and 200Mi memory are way too low for high-volume telemetry
|
||||
set {
|
||||
name = "clickhouse.resources.requests.cpu"
|
||||
value = "2000m"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "clickhouse.resources.requests.memory"
|
||||
value = "4Gi"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "clickhouse.resources.limits.cpu"
|
||||
value = "4000m"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "clickhouse.resources.limits.memory"
|
||||
value = "8Gi"
|
||||
}
|
||||
|
||||
# OTEL Collector resource configuration for loadtest
|
||||
set {
|
||||
name = "otelCollector.resources.requests.memory"
|
||||
value = "8Gi"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "otelCollector.resources.limits.memory"
|
||||
value = "12Gi"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "otelCollector.resources.requests.cpu"
|
||||
value = "1000m"
|
||||
}
|
||||
|
||||
set {
|
||||
name = "otelCollector.resources.limits.cpu"
|
||||
value = "4000m"
|
||||
}
|
||||
|
||||
# Only need 1 replica since we have 1 LoadBalancer endpoint
|
||||
set {
|
||||
name = "otelCollector.replicaCount"
|
||||
value = "1"
|
||||
}
|
||||
|
||||
# Uncomment to force redeployment
|
||||
#
|
||||
# set {
|
||||
# name = "redeployAt"
|
||||
# value = timestamp()
|
||||
# }
|
||||
|
||||
depends_on = [
|
||||
module.eks,
|
||||
kubernetes_storage_class_v1.gp3,
|
||||
helm_release.aws_load_balancer_controller
|
||||
]
|
||||
}
|
||||
|
||||
resource "kubernetes_secret" "otel_bearer_token" {
|
||||
metadata {
|
||||
name = "signoz-otel-bearer-token"
|
||||
namespace = "signoz"
|
||||
}
|
||||
|
||||
type = "Opaque"
|
||||
|
||||
data = {
|
||||
OTEL_BEARER_TOKEN = var.otel_bearer_token
|
||||
}
|
||||
}
|
||||
|
||||
resource "null_resource" "patch_otel_collector_bearer_env" {
|
||||
triggers = {
|
||||
token_hash = sha256(var.otel_bearer_token)
|
||||
helm_release = tostring(helm_release.signoz.metadata[0].revision)
|
||||
}
|
||||
|
||||
provisioner "local-exec" {
|
||||
command = <<-EOT
|
||||
set -euo pipefail
|
||||
kubectl -n signoz set env deployment/signoz-otel-collector \
|
||||
--containers=collector \
|
||||
--from=secret/${kubernetes_secret.otel_bearer_token.metadata[0].name}
|
||||
EOT
|
||||
}
|
||||
|
||||
depends_on = [
|
||||
helm_release.signoz,
|
||||
kubernetes_secret.otel_bearer_token
|
||||
]
|
||||
}
|
||||
|
||||
data "aws_lb" "signoz" {
|
||||
name = local.signoz_alb_name
|
||||
depends_on = [null_resource.wait_for_signoz_alb]
|
||||
}
|
||||
|
||||
|
||||
# Route53 records may require a 2nd apply if the load balancer changes in the EKS cluster.
|
||||
resource "aws_route53_record" "signoz" {
|
||||
zone_id = data.aws_route53_zone.dogfood.zone_id
|
||||
name = local.signoz_domain
|
||||
type = "A"
|
||||
|
||||
alias {
|
||||
name = data.aws_lb.signoz.dns_name
|
||||
zone_id = data.aws_lb.signoz.zone_id
|
||||
evaluate_target_health = true
|
||||
}
|
||||
}
|
||||
|
||||
resource "null_resource" "wait_for_signoz_alb" {
|
||||
depends_on = [helm_release.signoz]
|
||||
|
||||
triggers = {
|
||||
alb_name = local.signoz_alb_name
|
||||
run_id = timestamp()
|
||||
}
|
||||
|
||||
provisioner "local-exec" {
|
||||
command = <<-EOT
|
||||
set -euo pipefail
|
||||
for i in $(seq 1 60); do
|
||||
if aws elbv2 describe-load-balancers --names "${local.signoz_alb_name}" --region "${data.aws_region.current.region}" >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
echo "Timed out waiting for ALB ${local.signoz_alb_name}" >&2
|
||||
exit 1
|
||||
EOT
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_route53_record" "otlp" {
|
||||
zone_id = data.aws_route53_zone.dogfood.zone_id
|
||||
name = local.otlp_domain
|
||||
type = "A"
|
||||
|
||||
alias {
|
||||
name = data.aws_lb.signoz.dns_name
|
||||
zone_id = data.aws_lb.signoz.zone_id
|
||||
evaluate_target_health = true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
# OTEL Collector configuration overrides for production stability
|
||||
# These settings fix ClickHouse timeout issues by adding proper timeouts,
|
||||
# retry logic, and sending queues.
|
||||
|
||||
otelCollector:
|
||||
# Enable low cardinality exception grouping to reduce exception noise
|
||||
lowCardinalityExceptionGrouping: true
|
||||
|
||||
config:
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
bearertokenauth:
|
||||
scheme: "Bearer"
|
||||
token: ${env:OTEL_BEARER_TOKEN}
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
auth:
|
||||
authenticator: bearertokenauth
|
||||
http:
|
||||
auth:
|
||||
authenticator: bearertokenauth
|
||||
|
||||
exporters:
|
||||
# Logs exporter with balanced queue sizes for throughput and memory
|
||||
clickhouselogsexporter:
|
||||
timeout: 60s
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 5s
|
||||
max_interval: 30s
|
||||
max_elapsed_time: 300s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 10
|
||||
queue_size: 10000
|
||||
|
||||
# Traces exporter must handle spans after tail_sampling
|
||||
clickhousetraces:
|
||||
timeout: 120s
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 5s
|
||||
max_interval: 30s
|
||||
max_elapsed_time: 300s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 10
|
||||
queue_size: 10000
|
||||
|
||||
# Metadata exporter with increased timeout and retry support
|
||||
metadataexporter:
|
||||
timeout: 60s
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 5s
|
||||
max_interval: 30s
|
||||
max_elapsed_time: 300s
|
||||
|
||||
processors:
|
||||
# Optimized batch processor to send data more frequently
|
||||
# Smaller batches reduce memory pressure
|
||||
batch:
|
||||
send_batch_size: 2000
|
||||
timeout: 5s
|
||||
|
||||
# Memory limiter to prevent out-of-memory by applying back pressure
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 11264
|
||||
spike_limit_mib: 512
|
||||
|
||||
service:
|
||||
extensions:
|
||||
- health_check
|
||||
- bearertokenauth
|
||||
pipelines:
|
||||
traces:
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
@@ -0,0 +1,28 @@
|
||||
output "cluster_name" {
|
||||
description = "EKS cluster name"
|
||||
value = module.eks.cluster_name
|
||||
}
|
||||
|
||||
output "cluster_endpoint" {
|
||||
description = "EKS cluster endpoint"
|
||||
value = module.eks.cluster_endpoint
|
||||
}
|
||||
|
||||
output "cluster_certificate_authority_data" {
|
||||
description = "EKS cluster CA certificate"
|
||||
value = module.eks.cluster_certificate_authority_data
|
||||
}
|
||||
|
||||
output "configure_kubectl" {
|
||||
value = "aws eks update-kubeconfig --region ${data.aws_region.current.region} --name ${module.eks.cluster_name}"
|
||||
}
|
||||
|
||||
output "signoz_ui_url" {
|
||||
value = "https://${local.signoz_domain}"
|
||||
}
|
||||
|
||||
# Output for programmatic access - internal LoadBalancer hostname
|
||||
output "otel_collector_endpoint" {
|
||||
description = "Internal OTLP collector endpoint (https://host:port)"
|
||||
value = "https://${local.otlp_domain}:443"
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
NAMESPACE="${NAMESPACE:-signoz}"
|
||||
ENDPOINT="${OTLP_ENDPOINT:-otlp.signoz.dogfood.fleetdm.com:443}"
|
||||
TOKEN="${OTEL_BEARER_TOKEN:-}"
|
||||
DURATION="${DURATION:-15s}"
|
||||
IMAGE="${TELEMETRYGEN_IMAGE:-ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest}"
|
||||
|
||||
if [[ -z "${TOKEN}" ]]; then
|
||||
echo "OTEL_BEARER_TOKEN is required." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
kubectl -n "${NAMESPACE}" run signoz-telemetrygen \
|
||||
--rm -i --restart=Never \
|
||||
--image "${IMAGE}" \
|
||||
--command -- /telemetrygen traces \
|
||||
--otlp-endpoint "${ENDPOINT}" \
|
||||
--otlp-header "authorization=\"Bearer ${TOKEN}\"" \
|
||||
--duration "${DURATION}"
|
||||
Reference in New Issue
Block a user