Replace the hardcoded FLEET_MYSQL_MAX_OPEN_CONNS=10 in the loadtest terraform (both the root ecs.tf stack and the infra/ stack) with a mysql_max_open_conns variable, applied to both the writer and read-replica pools. Also document per-tier connection pool sizing in the AWS reference architecture: 20 for R-class instances (25k+ hosts), 10 for the smaller T-class instances (5k/10k hosts) whose low default max_connections (~90/135) leaves little headroom. <!-- Add the related story/sub-task/bug number, like Resolves #123, or remove if NA --> **Related issue:** Resolves #44802 <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added a configurable setting for MySQL connection limits in the load testing environment. * Connection settings now use a shared input value instead of fixed defaults, making it easier to tune database load behavior across container types. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Victor Lyuboslavsky
@@ -185,9 +185,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render
|
||||
|
||||
###### [Up to 5000 hosts](https://calculator.aws/#/estimate?id=cc26267f829536383e9b1b449ca0c56f0e844084)
|
||||
|
||||
| Fleet instances | CPU Units | RAM |
|
||||
| --------------- | -------------- | --- |
|
||||
| 6 Fargate tasks | 1024 CPU Units | 4GB |
|
||||
| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS |
|
||||
| --------------- | -------------- | --- | -------------------------- |
|
||||
| 6 Fargate tasks | 1024 CPU Units | 4GB | 10 |
|
||||
|
||||
| Dependencies | Version | Instance type | Nodes |
|
||||
| ------------ |-------------------------| --------------- | ----- |
|
||||
@@ -196,9 +196,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render
|
||||
|
||||
###### [Up to 10000 hosts](https://calculator.aws/#/estimate?id=5ea231525450b1cd4fa847f4564351d2c17d2ee2)
|
||||
|
||||
| Fleet instances | CPU Units | RAM |
|
||||
| --------------- | -------------- | --- |
|
||||
| 8 Fargate tasks | 1024 CPU Units | 4GB |
|
||||
| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS |
|
||||
| --------------- | -------------- | --- | -------------------------- |
|
||||
| 8 Fargate tasks | 1024 CPU Units | 4GB | 10 |
|
||||
|
||||
| Dependencies | Version | Instance type | Nodes |
|
||||
| ------------ |-------------------------| ---------------- | ----- |
|
||||
@@ -207,9 +207,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render
|
||||
|
||||
###### [Up to 25000 hosts](https://calculator.aws/#/estimate?id=4700e7a36ef34b84ae9ad4f444690f1df2ca3753)
|
||||
|
||||
| Fleet instances | CPU Units | RAM |
|
||||
| ---------------- | -------------- | --- |
|
||||
| 10 Fargate tasks | 1024 CPU Units | 4GB |
|
||||
| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS |
|
||||
| ---------------- | -------------- | --- | -------------------------- |
|
||||
| 10 Fargate tasks | 1024 CPU Units | 4GB | 20 |
|
||||
|
||||
| Dependencies | Version | Instance type | Nodes |
|
||||
| ------------ |-------------------------| --------------- | ----- |
|
||||
@@ -218,9 +218,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render
|
||||
|
||||
###### [Up to 50000 hosts](https://calculator.aws/#/estimate?id=3887d782a8f6cfeb9e0463686b5629aeb4cd678e)
|
||||
|
||||
| Fleet instances | CPU Units | RAM |
|
||||
| ---------------- | -------------- | --- |
|
||||
| 15 Fargate tasks | 1024 CPU Units | 4GB |
|
||||
| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS |
|
||||
| ---------------- | -------------- | --- | -------------------------- |
|
||||
| 15 Fargate tasks | 1024 CPU Units | 4GB | 20 |
|
||||
|
||||
| Dependencies | Version | Instance type | Nodes |
|
||||
| ------------ |-------------------------| --------------- | ----- |
|
||||
@@ -229,9 +229,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render
|
||||
|
||||
###### [Up to 80000 hosts](https://calculator.aws/#/estimate?id=7d0dee9241aff55fc733a9eead816baea14aee21)
|
||||
|
||||
| Fleet instances | CPU Units | RAM |
|
||||
| ---------------- | -------------- | --- |
|
||||
| 20 Fargate tasks | 1024 CPU Units | 4GB |
|
||||
| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS |
|
||||
| ---------------- | -------------- | --- | -------------------------- |
|
||||
| 20 Fargate tasks | 1024 CPU Units | 4GB | 20 |
|
||||
|
||||
| Dependencies | Version | Instance type | Nodes |
|
||||
| ------------ |-------------------------| --------------- | ----- |
|
||||
@@ -240,9 +240,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render
|
||||
|
||||
###### [Up to 100000 hosts](https://calculator.aws/#/estimate?id=cd17a0dda5e1ee5f919ac0a2a0ea8a6e1557e307)
|
||||
|
||||
| Fleet instances | CPU Units | RAM |
|
||||
| ---------------- | -------------- | --- |
|
||||
| 25 Fargate tasks | 1024 CPU Units | 4GB |
|
||||
| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS |
|
||||
| ---------------- | -------------- | --- | -------------------------- |
|
||||
| 25 Fargate tasks | 1024 CPU Units | 4GB | 20 |
|
||||
|
||||
| Dependencies | Version | Instance type | Nodes |
|
||||
| ------------ |-------------------------| --------------- | ----- |
|
||||
@@ -251,9 +251,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render
|
||||
|
||||
###### [Up to 150000 hosts](https://calculator.aws/#/estimate?id=a53e31063df9e5941b0c4b019b03ca2bd226fd48)
|
||||
|
||||
| Fleet instances | CPU Units | RAM |
|
||||
| ---------------- | -------------- | --- |
|
||||
| 40 Fargate tasks | 1024 CPU Units | 4GB |
|
||||
| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS |
|
||||
| ---------------- | -------------- | --- | -------------------------- |
|
||||
| 40 Fargate tasks | 1024 CPU Units | 4GB | 20 |
|
||||
|
||||
| Dependencies | Version | Instance type | Nodes |
|
||||
| ------------ |-------------------------| --------------- | ----- |
|
||||
@@ -262,9 +262,9 @@ See https://fleetdm.com/docs/deploy/deploy-fleet#render
|
||||
|
||||
###### [Up to 300000 hosts](https://calculator.aws/#/estimate?id=1f54ccc80e27a78f192b0e9db02ab957eff0c26c)
|
||||
|
||||
| Fleet instances | CPU Units | RAM |
|
||||
| ---------------- | -------------- | --- |
|
||||
| 70 Fargate tasks | 1024 CPU Units | 4GB |
|
||||
| Fleet instances | CPU Units | RAM | FLEET_MYSQL_MAX_OPEN_CONNS |
|
||||
| ---------------- | -------------- | --- | -------------------------- |
|
||||
| 70 Fargate tasks | 1024 CPU Units | 4GB | 20 |
|
||||
|
||||
| Dependencies | Version | Instance type | Nodes |
|
||||
| ------------ |-------------------------| ---------------- | ----- |
|
||||
|
||||
@@ -150,7 +150,7 @@ resource "aws_ecs_task_definition" "backend" {
|
||||
},
|
||||
{
|
||||
name = "FLEET_MYSQL_MAX_OPEN_CONNS"
|
||||
value = "10"
|
||||
value = tostring(var.mysql_max_open_conns)
|
||||
},
|
||||
{
|
||||
name = "FLEET_MYSQL_READ_REPLICA_USERNAME"
|
||||
@@ -166,7 +166,7 @@ resource "aws_ecs_task_definition" "backend" {
|
||||
},
|
||||
{
|
||||
name = "FLEET_MYSQL_READ_REPLICA_MAX_OPEN_CONNS"
|
||||
value = "10"
|
||||
value = tostring(var.mysql_max_open_conns)
|
||||
},
|
||||
{
|
||||
name = "FLEET_REDIS_ADDRESS"
|
||||
|
||||
@@ -38,8 +38,8 @@ locals {
|
||||
FLEET_FILESYSTEM_STATUS_LOG_FILE = "/dev/null"
|
||||
FLEET_OSQUERY_RESULT_LOG_PLUGIN = "filesystem"
|
||||
FLEET_FILESYSTEM_RESULT_LOG_FILE = "/dev/null"
|
||||
FLEET_MYSQL_MAX_OPEN_CONNS = "10"
|
||||
FLEET_MYSQL_READ_REPLICA_MAX_OPEN_CONNS = "10"
|
||||
FLEET_MYSQL_MAX_OPEN_CONNS = tostring(var.mysql_max_open_conns)
|
||||
FLEET_MYSQL_READ_REPLICA_MAX_OPEN_CONNS = tostring(var.mysql_max_open_conns)
|
||||
# 30 min: recycle connections often enough that pooled reader connections re-spread across replicas after a
|
||||
# replica reboot/failover, and proactively drop bad idles.
|
||||
FLEET_MYSQL_CONN_MAX_LIFETIME = "1800"
|
||||
|
||||
@@ -43,6 +43,17 @@ variable "database_instance_count" {
|
||||
}
|
||||
}
|
||||
|
||||
variable "mysql_max_open_conns" {
|
||||
description = "Max open MySQL connections per Fleet container, applied to both the writer and read-replica pools. A single Aurora instance sees roughly fleet_task_count * mysql_max_open_conns, up to 2x that on one instance during a failover or with no read replicas (when reader traffic falls back to the writer)."
|
||||
type = number
|
||||
default = 10
|
||||
|
||||
validation {
|
||||
condition = var.mysql_max_open_conns > 0
|
||||
error_message = "var.mysql_max_open_conns must be greater than 0 (0 means unlimited in database/sql, which can exhaust Aurora max_connections)."
|
||||
}
|
||||
}
|
||||
|
||||
variable "redis_instance_size" {
|
||||
description = "The instance size for Elasticache nodes"
|
||||
type = string
|
||||
|
||||
@@ -32,6 +32,17 @@ variable "db_instance_type" {
|
||||
default = "db.r6g.4xlarge"
|
||||
}
|
||||
|
||||
variable "mysql_max_open_conns" {
|
||||
description = "Max open MySQL connections per Fleet container, applied to both the writer and read-replica pools. A single Aurora instance sees roughly fleet_containers * mysql_max_open_conns, up to 2x that on one instance during a failover or with no read replicas (when reader traffic falls back to the writer)."
|
||||
type = number
|
||||
default = 10
|
||||
|
||||
validation {
|
||||
condition = var.mysql_max_open_conns > 0
|
||||
error_message = "var.mysql_max_open_conns must be greater than 0 (0 means unlimited in database/sql, which can exhaust Aurora max_connections)."
|
||||
}
|
||||
}
|
||||
|
||||
variable "redis_instance_type" {
|
||||
description = "the redis instance type to use in loadtesting. default is cache.m6g.large"
|
||||
type = string
|
||||
|
||||
Reference in New Issue
Block a user