From 426de4e5a85e6c70e2a0705a17e0a137c50a87f9 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky <2685025+getvictor@users.noreply.github.com> Date: Sat, 4 Jul 2026 06:48:22 +0100 Subject: [PATCH] Loadtest: make MySQL max open conns tunable (#44802) (#48472) Replace the hardcoded FLEET_MYSQL_MAX_OPEN_CONNS=10 in the loadtest terraform (both the root ecs.tf stack and the infra/ stack) with a mysql_max_open_conns variable, applied to both the writer and read-replica pools. Also document per-tier connection pool sizing in the AWS reference architecture: 20 for R-class instances (25k+ hosts), 10 for the smaller T-class instances (5k/10k hosts) whose low default max_connections (~90/135) leaves little headroom. **Related issue:** Resolves #44802 ## Summary by CodeRabbit * **New Features** * Added a configurable setting for MySQL connection limits in the load testing environment. * Connection settings now use a shared input value instead of fixed defaults, making it easier to tune database load behavior across container types. --- docs/Deploy/Reference-Architectures.md | 48 +++++++++---------- infrastructure/loadtesting/terraform/ecs.tf | 4 +- .../loadtesting/terraform/infra/locals.tf | 4 +- .../loadtesting/terraform/infra/variables.tf | 11 +++++ .../loadtesting/terraform/variables.tf | 11 +++++ 5 files changed, 50 insertions(+), 28 deletions(-) diff --git a/docs/Deploy/Reference-Architectures.md b/docs/Deploy/Reference-Architectures.md index fffe978283..5cf3e329e2 100644 --- a/docs/Deploy/Reference-Architectures.md +++ b/docs/Deploy/Reference-Architectures.md @@ -185,9 +185,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render ###### [Up to 5000 hosts](https://calculator.aws/#/estimate?id=cc26267f829536383e9b1b449ca0c56f0e844084) -| Fleet instances | CPU Units | RAM | -| --------------- | -------------- | --- | -| 6 Fargate tasks | 1024 CPU Units | 4GB | +| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS | +| --------------- | -------------- | --- | -------------------------- | +| 6 Fargate tasks | 1024 CPU Units | 4GB | 10 | | Dependencies | Version | Instance type | Nodes | | ------------ |-------------------------| --------------- | ----- | @@ -196,9 +196,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render ###### [Up to 10000 hosts](https://calculator.aws/#/estimate?id=5ea231525450b1cd4fa847f4564351d2c17d2ee2) -| Fleet instances | CPU Units | RAM | -| --------------- | -------------- | --- | -| 8 Fargate tasks | 1024 CPU Units | 4GB | +| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS | +| --------------- | -------------- | --- | -------------------------- | +| 8 Fargate tasks | 1024 CPU Units | 4GB | 10 | | Dependencies | Version | Instance type | Nodes | | ------------ |-------------------------| ---------------- | ----- | @@ -207,9 +207,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render ###### [Up to 25000 hosts](https://calculator.aws/#/estimate?id=4700e7a36ef34b84ae9ad4f444690f1df2ca3753) -| Fleet instances | CPU Units | RAM | -| ---------------- | -------------- | --- | -| 10 Fargate tasks | 1024 CPU Units | 4GB | +| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS | +| ---------------- | -------------- | --- | -------------------------- | +| 10 Fargate tasks | 1024 CPU Units | 4GB | 20 | | Dependencies | Version | Instance type | Nodes | | ------------ |-------------------------| --------------- | ----- | @@ -218,9 +218,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render ###### [Up to 50000 hosts](https://calculator.aws/#/estimate?id=3887d782a8f6cfeb9e0463686b5629aeb4cd678e) -| Fleet instances | CPU Units | RAM | -| ---------------- | -------------- | --- | -| 15 Fargate tasks | 1024 CPU Units | 4GB | +| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS | +| ---------------- | -------------- | --- | -------------------------- | +| 15 Fargate tasks | 1024 CPU Units | 4GB | 20 | | Dependencies | Version | Instance type | Nodes | | ------------ |-------------------------| --------------- | ----- | @@ -229,9 +229,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render ###### [Up to 80000 hosts](https://calculator.aws/#/estimate?id=7d0dee9241aff55fc733a9eead816baea14aee21) -| Fleet instances | CPU Units | RAM | -| ---------------- | -------------- | --- | -| 20 Fargate tasks | 1024 CPU Units | 4GB | +| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS | +| ---------------- | -------------- | --- | -------------------------- | +| 20 Fargate tasks | 1024 CPU Units | 4GB | 20 | | Dependencies | Version | Instance type | Nodes | | ------------ |-------------------------| --------------- | ----- | @@ -240,9 +240,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render ###### [Up to 100000 hosts](https://calculator.aws/#/estimate?id=cd17a0dda5e1ee5f919ac0a2a0ea8a6e1557e307) -| Fleet instances | CPU Units | RAM | -| ---------------- | -------------- | --- | -| 25 Fargate tasks | 1024 CPU Units | 4GB | +| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS | +| ---------------- | -------------- | --- | -------------------------- | +| 25 Fargate tasks | 1024 CPU Units | 4GB | 20 | | Dependencies | Version | Instance type | Nodes | | ------------ |-------------------------| --------------- | ----- | @@ -251,9 +251,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render ###### [Up to 150000 hosts](https://calculator.aws/#/estimate?id=a53e31063df9e5941b0c4b019b03ca2bd226fd48) -| Fleet instances | CPU Units | RAM | -| ---------------- | -------------- | --- | -| 40 Fargate tasks | 1024 CPU Units | 4GB | +| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS | +| ---------------- | -------------- | --- | -------------------------- | +| 40 Fargate tasks | 1024 CPU Units | 4GB | 20 | | Dependencies | Version | Instance type | Nodes | | ------------ |-------------------------| --------------- | ----- | @@ -262,9 +262,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render ###### [Up to 300000 hosts](https://calculator.aws/#/estimate?id=1f54ccc80e27a78f192b0e9db02ab957eff0c26c) -| Fleet instances | CPU Units | RAM | -| ---------------- | -------------- | --- | -| 70 Fargate tasks | 1024 CPU Units | 4GB | +| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS | +| ---------------- | -------------- | --- | -------------------------- | +| 70 Fargate tasks | 1024 CPU Units | 4GB | 20 | | Dependencies | Version | Instance type | Nodes | | ------------ |-------------------------| ---------------- | ----- | diff --git a/infrastructure/loadtesting/terraform/ecs.tf b/infrastructure/loadtesting/terraform/ecs.tf index 834079cb39..0e86dd9936 100644 --- a/infrastructure/loadtesting/terraform/ecs.tf +++ b/infrastructure/loadtesting/terraform/ecs.tf @@ -150,7 +150,7 @@ resource "aws_ecs_task_definition" "backend" { }, { name = "FLEET_MYSQL_MAX_OPEN_CONNS" - value = "10" + value = tostring(var.mysql_max_open_conns) }, { name = "FLEET_MYSQL_READ_REPLICA_USERNAME" @@ -166,7 +166,7 @@ resource "aws_ecs_task_definition" "backend" { }, { name = "FLEET_MYSQL_READ_REPLICA_MAX_OPEN_CONNS" - value = "10" + value = tostring(var.mysql_max_open_conns) }, { name = "FLEET_REDIS_ADDRESS" diff --git a/infrastructure/loadtesting/terraform/infra/locals.tf b/infrastructure/loadtesting/terraform/infra/locals.tf index be87079ca6..657759630d 100644 --- a/infrastructure/loadtesting/terraform/infra/locals.tf +++ b/infrastructure/loadtesting/terraform/infra/locals.tf @@ -38,8 +38,8 @@ locals { FLEET_FILESYSTEM_STATUS_LOG_FILE = "/dev/null" FLEET_OSQUERY_RESULT_LOG_PLUGIN = "filesystem" FLEET_FILESYSTEM_RESULT_LOG_FILE = "/dev/null" - FLEET_MYSQL_MAX_OPEN_CONNS = "10" - FLEET_MYSQL_READ_REPLICA_MAX_OPEN_CONNS = "10" + FLEET_MYSQL_MAX_OPEN_CONNS = tostring(var.mysql_max_open_conns) + FLEET_MYSQL_READ_REPLICA_MAX_OPEN_CONNS = tostring(var.mysql_max_open_conns) # 30 min: recycle connections often enough that pooled reader connections re-spread across replicas after a # replica reboot/failover, and proactively drop bad idles. FLEET_MYSQL_CONN_MAX_LIFETIME = "1800" diff --git a/infrastructure/loadtesting/terraform/infra/variables.tf b/infrastructure/loadtesting/terraform/infra/variables.tf index 8cb41a2718..8f342b23cf 100644 --- a/infrastructure/loadtesting/terraform/infra/variables.tf +++ b/infrastructure/loadtesting/terraform/infra/variables.tf @@ -43,6 +43,17 @@ variable "database_instance_count" { } } +variable "mysql_max_open_conns" { + description = "Max open MySQL connections per Fleet container, applied to both the writer and read-replica pools. A single Aurora instance sees roughly fleet_task_count * mysql_max_open_conns, up to 2x that on one instance during a failover or with no read replicas (when reader traffic falls back to the writer)." + type = number + default = 10 + + validation { + condition = var.mysql_max_open_conns > 0 + error_message = "var.mysql_max_open_conns must be greater than 0 (0 means unlimited in database/sql, which can exhaust Aurora max_connections)." + } +} + variable "redis_instance_size" { description = "The instance size for Elasticache nodes" type = string diff --git a/infrastructure/loadtesting/terraform/variables.tf b/infrastructure/loadtesting/terraform/variables.tf index af63be2af2..e197e522ff 100644 --- a/infrastructure/loadtesting/terraform/variables.tf +++ b/infrastructure/loadtesting/terraform/variables.tf @@ -32,6 +32,17 @@ variable "db_instance_type" { default = "db.r6g.4xlarge" } +variable "mysql_max_open_conns" { + description = "Max open MySQL connections per Fleet container, applied to both the writer and read-replica pools. A single Aurora instance sees roughly fleet_containers * mysql_max_open_conns, up to 2x that on one instance during a failover or with no read replicas (when reader traffic falls back to the writer)." + type = number + default = 10 + + validation { + condition = var.mysql_max_open_conns > 0 + error_message = "var.mysql_max_open_conns must be greater than 0 (0 means unlimited in database/sql, which can exhaust Aurora max_connections)." + } +} + variable "redis_instance_type" { description = "the redis instance type to use in loadtesting. default is cache.m6g.large" type = string