diff --git a/.github/workflows/loadtest-osquery-perf.yml b/.github/workflows/loadtest-osquery-perf.yml index 87530cf25c..bfe6ff3c8e 100644 --- a/.github/workflows/loadtest-osquery-perf.yml +++ b/.github/workflows/loadtest-osquery-perf.yml @@ -7,20 +7,19 @@ on: description: "Terraform workspace that you will be deploying to." type: string required: true - tag: - description: "Tag for osquery-perf deployment" - type: string - default: "v4.72.0" - required: true - git_branch: - description: "git branch for osquery-perf deployment" + git_tag_branch: + description: "git Tag or Branch to use for osquery-perf deployment" type: string default: "main" required: true loadtest_containers: - description: "Count of osquery-perf tasks to run" + description: "Deploys osquery-perf containers all at once. Total number of osquery-perf tasks to run (should be a multiple of 8, if setting loadtest_containers_starting_index). This is also used as the end index in enroll.sh" type: string required: true + loadtest_containers_starting_index: + description: "Optional: Starting Index for enroll.sh (should be a multiple of 8)." + type: string + required: false extra_flags: description: "Extra flags for osquery-perf. Example: [\"--orbit_prob\", \"0.0\"]" type: string @@ -52,8 +51,7 @@ env: TF_ACTIONS_WORKING_DIR: infrastructure/loadtesting/terraform/osquery_perf TF_VAR_extra_flags: "${{ inputs.extra_flags || '[]' }}" TF_VAR_loadtest_containers: "${{ inputs.loadtest_containers }}" - TF_VAR_tag: "${{ inputs.tag }}" - TF_VAR_git_branch: "${{ inputs.git_branch }}" + TF_VAR_git_tag_branch: "${{ inputs.git_tag_branch }}" permissions: id-token: write @@ -145,7 +143,12 @@ jobs: if [[ `terraform workspace show` = "${{ inputs.terraform_workspace }}" ]]; then echo "TERRAFORM WORKSPACE: MATCHES - ${{ inputs.terraform_workspace }}" - terraform apply -auto-approve + if [[ ${{ inputs.loadtest_containers_starting_index}} -gt "0" ]]; + then + ./enroll.sh ${{ inputs.git_tag_branch }} ${{ inputs.loadtest_containers_starting_index}} ${{ inputs.loadtest_containers }} + else + terraform apply -auto-approve + fi else echo "TERRAFORM WORKSPACE: DOES NOT MATCH INPUT - ${{ inputs.terraform_workspace }}" fi @@ -202,4 +205,4 @@ jobs: fi else echo "TERRAFORM WORKSPACE: DOES NOT MATCH INPUT - ${{ inputs.terraform_workspace }}" - fi \ No newline at end of file + fi diff --git a/infrastructure/loadtesting/terraform/osquery_perf/.header.md b/infrastructure/loadtesting/terraform/osquery_perf/.header.md index 212e25188a..38a77d16ec 100644 --- a/infrastructure/loadtesting/terraform/osquery_perf/.header.md +++ b/infrastructure/loadtesting/terraform/osquery_perf/.header.md @@ -9,7 +9,7 @@ Although deployments through the github action should be prioritized, for manual - Docker - Go -# Deploy with Github Actions (Coming Soon) +# Deploy with Github Actions 1. [Navigate to the github action](https://github.com/fleetdm/fleet/actions/workflows/loadtest-osquery-perf.yml) @@ -60,7 +60,7 @@ Although deployments through the github action should be prioritized, for manual > Note: Terraform will prompt you for confirmation to trigger the deployment. If everything looks ok, submitting `yes` will trigger the deployment. ```sh - terraform apply -var=tag=v4.73.0 -var=git_branch=fleet-v4.73.0 + terraform apply -var=git_tag_branch=fleet-v4.73.0 ``` or, you can add the additional supported terraform variables, to overwrite the default values. You can choose which ones are included/overwritten. If a variable is not defined, the default value configured in [./variables.tf](variables.tf) is used. @@ -68,9 +68,15 @@ Although deployments through the github action should be prioritized, for manual Below is an example with all available variables. ```sh - terraform apply -var=tag=v4.73.0 -var=git_branch=fleet-v4.73.0 -var=loadtest_containers=20 -var=extra_flags=["--orbit_prob", "0.0"] + terraform apply -var=git_tag_branch=fleet-v4.73.0 -var=loadtest_containers=20 -var=extra_flags=["--orbit_prob", "0.0"] ``` +6. If you'd like to deploy osquery_perf tasks in batches, you can now run the original `enroll.sh` script, from the osquery_perf directory. The script will deploy in batches of 8, every 60 seconds, so it's recommended to set your starting index and max number of osquery perf containers as a multiple of 8. + + ```sh + ./enroll.sh + ``` + # Destroy osquery perf manually 1. Clone the repository (if not already cloned) diff --git a/infrastructure/loadtesting/terraform/osquery_perf/README.md b/infrastructure/loadtesting/terraform/osquery_perf/README.md index 3f165e7d5a..6b14c8fa0d 100644 --- a/infrastructure/loadtesting/terraform/osquery_perf/README.md +++ b/infrastructure/loadtesting/terraform/osquery_perf/README.md @@ -9,7 +9,7 @@ Although deployments through the github action should be prioritized, for manual - Docker - Go -# Deploy with Github Actions (Coming Soon) +# Deploy with Github Actions 1. [Navigate to the github action](https://github.com/fleetdm/fleet/actions/workflows/loadtest-osquery-perf.yml) @@ -60,7 +60,7 @@ Although deployments through the github action should be prioritized, for manual > Note: Terraform will prompt you for confirmation to trigger the deployment. If everything looks ok, submitting `yes` will trigger the deployment. ```sh - terraform apply -var=tag=v4.73.0 -var=git_branch=fleet-v4.73.0 + terraform apply -var=git_tag_branch=fleet-v4.73.0 ``` or, you can add the additional supported terraform variables, to overwrite the default values. You can choose which ones are included/overwritten. If a variable is not defined, the default value configured in [./variables.tf](variables.tf) is used. @@ -68,9 +68,15 @@ Although deployments through the github action should be prioritized, for manual Below is an example with all available variables. ```sh - terraform apply -var=tag=v4.73.0 -var=git_branch=fleet-v4.73.0 -var=loadtest_containers=20 -var=extra_flags=["--orbit_prob", "0.0"] + terraform apply -var=git_tag_branch=fleet-v4.73.0 -var=loadtest_containers=20 -var=extra_flags=["--orbit_prob", "0.0"] ``` +6. If you'd like to deploy osquery\_perf tasks in batches, you can now run the original `enroll.sh` script, from the osquery\_perf directory. The script will deploy in batches of 8, every 60 seconds, so it's recommended to set your starting index and max number of osquery perf containers as a multiple of 8. + + ```sh + ./enroll.sh + ``` + # Destroy osquery perf manually 1. Clone the repository (if not already cloned) @@ -114,7 +120,7 @@ terraform workspace delete | Name | Version | |------|---------| -| [aws](#provider\_aws) | 6.18.0 | +| [aws](#provider\_aws) | 6.20.0 | | [docker](#provider\_docker) | 3.6.2 | | [git](#provider\_git) | 2025.10.10 | | [terraform](#provider\_terraform) | n/a | @@ -135,7 +141,6 @@ terraform workspace delete | [aws_ecr_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_authorization_token) | data source | | [aws_ecr_repository.fleet](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecr_repository) | data source | | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | -| [docker_registry_image.dockerhub](https://registry.terraform.io/providers/kreuzwerker/docker/latest/docs/data-sources/registry_image) | data source | | [git_repository.tf](https://registry.terraform.io/providers/metio/git/2025.10.10/docs/data-sources/repository) | data source | | [terraform_remote_state.infra](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/data-sources/remote_state) | data source | | [terraform_remote_state.shared](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/data-sources/remote_state) | data source | @@ -145,9 +150,8 @@ terraform workspace delete | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [extra\_flags](#input\_extra\_flags) | Comma delimited list (string) for passing extra flags to osquery-perf containers | `list(string)` |
[
"--orbit_prob",
"0.0"
]
| no | -| [git\_branch](#input\_git\_branch) | The git branch to use to build loadtest containers. Only needed if docker tag doesn't match the git branch | `string` | `null` | no | +| [git\_tag\_branch](#input\_git\_tag\_branch) | The tag or git branch to use to build loadtest containers. | `string` | `null` | no | | [loadtest\_containers](#input\_loadtest\_containers) | Number of loadtest containers to deploy | `number` | `1` | no | -| [tag](#input\_tag) | The docker image tag to deploy. The image must exist in fleetdm/fleet docker repository | `any` | n/a | yes | ## Outputs diff --git a/infrastructure/loadtesting/terraform/osquery_perf/docker.tf b/infrastructure/loadtesting/terraform/osquery_perf/docker.tf index b3599d13a1..6f36cae79a 100644 --- a/infrastructure/loadtesting/terraform/osquery_perf/docker.tf +++ b/infrastructure/loadtesting/terraform/osquery_perf/docker.tf @@ -1,9 +1,5 @@ data "aws_ecr_authorization_token" "token" {} -data "docker_registry_image" "dockerhub" { - name = "fleetdm/fleet:${var.tag}" -} - data "aws_ecr_repository" "fleet" { name = local.customer } @@ -14,8 +10,9 @@ resource "docker_registry_image" "loadtest" { } resource "docker_image" "loadtest" { - name = "${data.aws_ecr_repository.fleet.repository_url}:loadtest-${local.loadtest_tag}-${split(":", data.docker_registry_image.dockerhub.sha256_digest)[1]}" + name = "${data.aws_ecr_repository.fleet.repository_url}:loadtest-${local.loadtest_tag}-${formatdate("YYYYMMDD-HHmmss", timestamp())}" keep_locally = true + force_remove = true build { context = "../docker/" dockerfile = "loadtest.Dockerfile" diff --git a/infrastructure/loadtesting/terraform/osquery_perf/enroll.sh b/infrastructure/loadtesting/terraform/osquery_perf/enroll.sh new file mode 100755 index 0000000000..806b7100f0 --- /dev/null +++ b/infrastructure/loadtesting/terraform/osquery_perf/enroll.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -e +# Script for enrolling osquery-perf hosts by `terraform apply`ing in increments of 8 `loadtest` containers. +# NOTE(lucas): This is the currently known configuration that won't tip the loadtest environment, +# but maybe in the future we can be more aggressive (and reduce enroll time). +# +# ./enroll.sh my-branch 8 240 + +BRANCH_NAME=$1 +START_INDEX=$2 +END_INDEX=$3 +INCREMENT=8 +SLEEP_TIME_SECONDS=60 + +if [ -z "$BRANCH_NAME" ]; then + echo "Missing BRANCH_NAME" +fi +if [ -z "$START_INDEX" ]; then + echo "Missing START_INDEX" +fi +if [ -z "$END_INDEX" ]; then + echo "Missing END_INDEX" +fi + +# We add this check to avoid terraform (error-prone) locking in case of typos. +# read -p "You will use BRANCH_NAME=$BRANCH_NAME. Continue? " + +set -x + +for (( c=$START_INDEX; c<=$END_INDEX; c+=$INCREMENT )); do + terraform apply -var git_tag_branch=$BRANCH_NAME -var loadtest_containers=$c -auto-approve + sleep $SLEEP_TIME_SECONDS +done diff --git a/infrastructure/loadtesting/terraform/osquery_perf/locals.tf b/infrastructure/loadtesting/terraform/osquery_perf/locals.tf index 35e78cffba..94fe90c43a 100644 --- a/infrastructure/loadtesting/terraform/osquery_perf/locals.tf +++ b/infrastructure/loadtesting/terraform/osquery_perf/locals.tf @@ -2,6 +2,6 @@ locals { customer = "fleet-${terraform.workspace}" loadtest_containers = var.loadtest_containers - fleet_image = var.tag - loadtest_tag = var.git_branch != null ? var.git_branch : var.tag -} \ No newline at end of file + # fleet_image = var.tag + loadtest_tag = var.git_tag_branch +} diff --git a/infrastructure/loadtesting/terraform/osquery_perf/main.tf b/infrastructure/loadtesting/terraform/osquery_perf/main.tf index a0759280c9..c644c4d023 100644 --- a/infrastructure/loadtesting/terraform/osquery_perf/main.tf +++ b/infrastructure/loadtesting/terraform/osquery_perf/main.tf @@ -1,4 +1,3 @@ - data "aws_caller_identity" "current" {} data "aws_region" "current" {} @@ -16,7 +15,7 @@ module "osquery_perf" { ecs_iam_role_arn = data.terraform_remote_state.infra.outputs.ecs_arn ecs_execution_iam_role_arn = data.terraform_remote_state.infra.outputs.ecs_execution_arn server_url = "http://${data.terraform_remote_state.infra.outputs.internal_alb_dns_name}" - osquery_perf_image = "${data.aws_ecr_repository.fleet.repository_url}:loadtest-${local.loadtest_tag}-${split(":", data.docker_registry_image.dockerhub.sha256_digest)[1]}" + osquery_perf_image = "${data.aws_ecr_repository.fleet.repository_url}:loadtest-${local.loadtest_tag}-${formatdate("YYYYMMDD-HHmmss", timestamp())}" extra_flags = var.extra_flags logging_options = data.terraform_remote_state.infra.outputs.logging_config enroll_secret_arn = data.terraform_remote_state.infra.outputs.enroll_secret_arn diff --git a/infrastructure/loadtesting/terraform/osquery_perf/variables.tf b/infrastructure/loadtesting/terraform/osquery_perf/variables.tf index 46a418dcbb..25bbc64afc 100644 --- a/infrastructure/loadtesting/terraform/osquery_perf/variables.tf +++ b/infrastructure/loadtesting/terraform/osquery_perf/variables.tf @@ -1,9 +1,5 @@ -variable "tag" { - description = "The docker image tag to deploy. The image must exist in fleetdm/fleet docker repository" -} - -variable "git_branch" { - description = "The git branch to use to build loadtest containers. Only needed if docker tag doesn't match the git branch" +variable "git_tag_branch" { + description = "The tag or git branch to use to build loadtest containers." type = string default = null }