Loadtesting - Enable Cloudfront (#31073)

# Added - Added kms.tf to support encrypting keys, specifically cloudfront keys. - Added template/cloudfront.tf.disabled for use in enabling cloudfront.- Modified ecs-iam.tf to support log-alb.tf, cloudfront.tf policies that are injected into `local.extra_execution_iam_policies` and `local.iam`. - Added log-alb.tf to enable logging alb, required by cloudfront.tf. # Changed - Modified ecs.tf to support adding of additional secrets from `local.secrets`. - Modified firehose.tf to support provider required updates for deprecated resource configurations. - Modified init.tf to support `> v5.0` of `hashicorp/aws` provider. - Modified locals.tf to add `extra_execution_iam_policies`, `iam`, `software_installers_kms_policy`, `extra_secrets`, secrets, and `cloudfront_key_basename`, to support cloudfront. - Modified readme.md with instructions on how to enable cloudfront.tf - Modified redis.tf to support provider required updates for deprecated resource configurations - Modified s3.tf to support kms keys and add kms iam. - Modified terraform version in .github/workflows/tfvalidate.yml - 1.9.0 -> 1.10.4
2025-07-21 16:41:06 -04:00
parent 9f34b49e4b
commit 3a112afdb6
12 changed files with 303 additions and 30 deletions
@@ -41,7 +41,7 @@ jobs:
      - name: Install terraform
        uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3
        with:
-          terraform_version: 1.9.0
+          terraform_version: 1.10.4
      # If we want to test more of these, consider using a matrix.  With a matrix of directories, all terraform modules could be fully tested and potentially in parallel.
      - name: Validate loadtesting
        working-directory: ./infrastructure/loadtesting/terraform
@@ -81,3 +81,40 @@ resource "aws_iam_role_policy_attachment" "attachment" {
  policy_arn = aws_iam_policy.main.arn
  role       = aws_iam_role.main.name
 }
+
+
+
+data "aws_iam_policy_document" "fleet-execution" {
+  // allow fleet application to obtain the database password from secrets manager
+  statement {
+    effect  = "Allow"
+    actions = ["secretsmanager:GetSecretValue"]
+    resources = [
+      aws_secretsmanager_secret.database_password_secret.arn,
+      aws_secretsmanager_secret.fleet_server_private_key.arn
+    ]
+  }
+}
+
+resource "aws_iam_role_policy_attachment" "execution_extras" {
+  for_each   = toset(local.extra_execution_iam_policies)
+  policy_arn = each.value
+  role       = aws_iam_role.execution.name
+}
+
+resource "aws_iam_policy" "execution" {
+  name        = local.iam.execution.policy_name
+  description = "IAM policy that Fleet application uses to define access to AWS resources"
+  policy      = data.aws_iam_policy_document.fleet-execution.json
+}
+
+resource "aws_iam_role_policy_attachment" "execution" {
+  policy_arn = aws_iam_policy.execution.arn
+  role       = aws_iam_role.execution.name
+}
+
+resource "aws_iam_role" "execution" {
+  name               = local.iam.execution.name
+  description        = "The execution role for Fleet in ECS"
+  assume_role_policy = data.aws_iam_policy_document.assume_role.json
+}
@@ -113,7 +113,7 @@ resource "aws_ecs_task_definition" "backend" {
            awslogs-stream-prefix = "fleet"
          }
        },
-        secrets = [
+        secrets = concat([
          {
            name      = "FLEET_MYSQL_PASSWORD"
            valueFrom = aws_secretsmanager_secret.database_password_secret.arn
@@ -130,7 +130,7 @@ resource "aws_ecs_task_definition" "backend" {
            name      = "FLEET_SERVER_PRIVATE_KEY"
            valueFrom = aws_secretsmanager_secret.fleet_server_private_key.arn
          }
-        ]
+        ], local.secrets)
        environment = concat([
          {
            name  = "FLEET_LOGGING_JSON"
@@ -1,6 +1,6 @@
 resource "aws_s3_bucket" "osquery-results" { #tfsec:ignore:aws-s3-encryption-customer-key tfsec:ignore:aws-s3-enable-bucket-logging tfsec:ignore:aws-s3-enable-versioning
  bucket = "${local.prefix}-loadtest-osquery-logs-archive"
-  
+
  # Allow destroy of non-empty buckets
  force_destroy = true

@@ -42,7 +42,7 @@ resource "aws_s3_bucket_public_access_block" "osquery-results" {

 resource "aws_s3_bucket" "osquery-status" { #tfsec:ignore:aws-s3-encryption-customer-key tfsec:ignore:aws-s3-enable-bucket-logging tfsec:ignore:aws-s3-enable-versioning
  bucket = "${local.prefix}-loadtest-osquery-status-archive"
-  
+
  # Allow destroy of non-empty buckets
  force_destroy = true

@@ -151,9 +151,9 @@ data "aws_iam_policy_document" "osquery_firehose_assume_role" {

 resource "aws_kinesis_firehose_delivery_stream" "osquery_results" {
  name        = "${local.prefix}-osquery_results"
-  destination = "s3"
+  destination = "extended_s3"

-  s3_configuration {
+  extended_s3_configuration {
    role_arn   = aws_iam_role.firehose-results.arn
    bucket_arn = aws_s3_bucket.osquery-results.arn
  }
@@ -161,9 +161,9 @@ resource "aws_kinesis_firehose_delivery_stream" "osquery_results" {

 resource "aws_kinesis_firehose_delivery_stream" "osquery_status" {
  name        = "${local.prefix}-osquery_status"
-  destination = "s3"
+  destination = "extended_s3"

-  s3_configuration {
+  extended_s3_configuration {
    role_arn   = aws_iam_role.firehose-status.arn
    bucket_arn = aws_s3_bucket.osquery-status.arn
  }
@@ -14,7 +14,7 @@ terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
-      version = "~> 4.32.0"
+      version = "~> 5.0"
    }
    docker = {
      source  = "kreuzwerker/docker"
@@ -0,0 +1,12 @@
+resource "aws_kms_key" "customer_data_key" {
+  description = "key used to encrypt sensitive data stored in terraform"
+}
+
+resource "aws_kms_alias" "alias" {
+  name          = "alias/${terraform.workspace}-terraform-encrypted"
+  target_key_id = aws_kms_key.customer_data_key.id
+}
+
+output "kms_key_id" {
+  value = aws_kms_key.customer_data_key.id
+}
@@ -1,6 +1,20 @@
 locals {
  name   = "fleetdm-${terraform.workspace}"
  prefix = "fleet-${terraform.workspace}"
+  extra_execution_iam_policies = concat(
+    # module.cloudfront-software-installers.extra_execution_iam_policies,
+    []
+  )
+  iam = {
+    role = {
+      name        = "${terraform.workspace}-role"
+      policy_name = "${terraform.workspace}-iam-policy"
+    }
+    execution = {
+      name        = "${terraform.workspace}-execution-role"
+      policy_name = "${terraform.workspace}-iam-policy-execution"
+    }
+  }
  additional_env_vars = [for k, v in merge({
    "FLEET_VULNERABILITIES_DATABASES_PATH" : "/home/fleet"
    "FLEET_OSQUERY_ENABLE_ASYNC_HOST_PROCESSING" : "false"
@@ -19,5 +33,24 @@ locals {
    "10.255.2.0/24",
    "10.255.3.0/24",
  ]
-
-}
+  software_installers_kms_policy = [{
+    sid = "AllowSoftwareInstallersKMSAccess"
+    actions = [
+      "kms:ReEncrypt*",
+      "kms:GenerateDataKey*",
+      "kms:Encrypt*",
+      "kms:Describe*",
+      "kms:Decrypt*"
+    ]
+    resources = [aws_kms_key.software_installers.arn]
+    effect    = "Allow"
+  }]
+  extra_secrets = merge(
+    # module.cloudfront-software-installers.extra_secrets
+  )
+  secrets = [for k, v in local.extra_secrets : {
+    name      = k
+    valueFrom = v
+  }]
+  cloudfront_key_basename = "cloudfront"
+}
@@ -0,0 +1,5 @@
+module "logging_alb" {
+  source        = "github.com/fleetdm/fleet-terraform/addons/logging-alb?depth=1&ref=tf-mod-addon-logging-alb-v1.3.0"
+  prefix        = terraform.workspace
+  enable_athena = true
+}
@@ -74,6 +74,134 @@ This has an impact on real devices because they will not be notified of any comm
 - `-mdm_prob 1.0`
 - `-mdm_scep_challenge` set to the same value as `FLEET_MDM_APPLE_SCEP_CHALLENGE` above.

+### Enabling Cloudfront
+
+> Do not commit your `BRANCH_NAME` if any files exist in `resources/TERRAFORM_WORKSPACE/` without a .encrypted extension.
+> This step assumes that you've already successfully executed terraform apply and have a `kms_key_id` output.
+
+1. Under the loadtesting directory, create directory `resources/TERRAFORM_WORKSPACE/`. Your `TERRAFORM_WORKSPACE` value can be retrieved with `terraform workspace show`.
+
+2. Change directory to `resources/TERRAFORM_WORKSPACE/`
+
+3. Create your keys
+
+```
+openssl genrsa -out cloudfront.key 2048
+openssl rsa -pubout -in cloudfront.key -out cloudfront.pem
+```
+
+4. Create `encrypt.sh` (store the script in in the terraform directory, two directories up `../..`)
+
+```
+#!/bin/bash
+
+set -e
+
+function usage() {
+	cat <<-EOUSAGE
+	
+	Usage: $(basename ${0}) <KMS_KEY_ID> <SOURCE> <DESTINATION> [AWS_PROFILE]
+	
+		This script encrypts an plaintext file from SOURCE into an
+		AWS KMS encrypted DESTINATION file.  Optionally you
+		may provide the AWS_PROFILE you wish to use to run the aws kms
+		commands.
+
+	EOUSAGE
+	exit 1
+}
+
+[ $# -lt 3 ] && usage
+
+if [ -n "${4}" ]; then
+	export AWS_PROFILE=${4}
+fi
+
+aws kms encrypt --key-id "${1:?}" --plaintext fileb://<(cat "${2:?}") --output text --query CiphertextBlob > "${3:?}"
+```
+
+5. Encrypt the objects using `encrypt.sh`
+
+```
+for i in *; do ../../encrypt.sh <KMS_KEY_ID> $i $i.encrypted; done
+for i in *.encrypted; do rm ${i/.encrypted/}; done
+```
+
+6. Change back to the terraform directory (two directories up ../..)
+
+7. If the name of your public/private cloudfront key is not `cloudfront.pem|.key`, update `locals.tf`
+
+```
+# Set the following variable value to the base name of your cloudfront key, no extension.
+cloudfront_key_basename = cloudfront
+```
+
+8. Copy and make `cloudfront.tf`
+
+```
+cp template/cloudfront.tf.disabled cloudfront.tf
+```
+
+9. In `locals.tf` uncomment the following line, under `extra_secrets`
+
+```
+module.cloudfront-software-installers.extra_secrets,
+```
+
+Example: You should end up with something that looks like the following block
+
+```
+  extra_secrets = merge(
+    module.cloudfront-software-installers.extra_secrets
+  )
+```
+
+10. Initialize terraform and upgrade any necessary dependencies
+
+```
+terraform init -upgrade
+```
+
+11. Apply the terraform
+
+```
+terraform apply -var tag=BRANCH_NAME
+```
+
+12. In `locals.tf` uncomment the following line, under `extra_execution_iam_policies`.
+
+```
+module.cloudfront-software-installers.extra_execution_iam_policies,
+```
+
+You should end up with something that looks like this.
+
+```
+  extra_execution_iam_policies = concat(
+    module.cloudfront-software-installers.extra_execution_iam_policies,
+    []
+  )
+```
+
+13. Apply the terraform
+
+```
+terraform apply -var tag=BRANCH_NAME
+```
+
+14. Cloudfront should now be enabled.
+
+15. If you had previously uploaded any software installers, they need to be re-encrypted by finding and targeting your bucket with the following commands.
+
+```
+# List buckets matching your BRANCH_NAME
+aws s3 ls | grep BRANCH_NAME
+
+# Replace <bucket-name> with the software_instalelrs bucket name.
+aws s3 cp s3://<bucket-name>/ s3://<bucket-name>/ --recursive
+```
+
+
 ### Running a loadtest

 We run simulated hosts in containers of 500 at a time. Once the infrastructure is running, you can run the following command:
@@ -1,20 +1,20 @@
 resource "aws_elasticache_replication_group" "default" {
-  availability_zones         = ["us-east-2a", "us-east-2b", "us-east-2c"]
-  engine                     = "redis"
-  parameter_group_name       = aws_elasticache_parameter_group.default.id
-  subnet_group_name          = data.terraform_remote_state.shared.outputs.vpc.elasticache_subnet_group_name
-  security_group_ids         = [aws_security_group.redis.id, aws_security_group.backend.id]
-  replication_group_id       = "${local.prefix}-redis"
-  num_cache_clusters         = 3
-  node_type                  = var.redis_instance_type
-  engine_version             = "6.2"
-  port                       = "6379"
-  snapshot_retention_limit   = 0
-  automatic_failover_enabled = true
-  at_rest_encryption_enabled = false #tfsec:ignore:aws-elasticache-enable-at-rest-encryption
-  transit_encryption_enabled = false #tfsec:ignore:aws-elasticache-enable-in-transit-encryption
-  apply_immediately          = true
-  description                = "${local.prefix}-redis"
+  preferred_cache_cluster_azs = ["us-east-2a", "us-east-2b", "us-east-2c"]
+  engine                      = "redis"
+  parameter_group_name        = aws_elasticache_parameter_group.default.id
+  subnet_group_name           = data.terraform_remote_state.shared.outputs.vpc.elasticache_subnet_group_name
+  security_group_ids          = [aws_security_group.redis.id, aws_security_group.backend.id]
+  replication_group_id        = "${local.prefix}-redis"
+  num_cache_clusters          = 3
+  node_type                   = var.redis_instance_type
+  engine_version              = "6.2"
+  port                        = "6379"
+  snapshot_retention_limit    = 0
+  automatic_failover_enabled  = true
+  at_rest_encryption_enabled  = false #tfsec:ignore:aws-elasticache-enable-at-rest-encryption
+  transit_encryption_enabled  = false #tfsec:ignore:aws-elasticache-enable-in-transit-encryption
+  apply_immediately           = true
+  description                 = "${local.prefix}-redis"

 }

@@ -13,6 +13,30 @@ data "aws_iam_policy_document" "software_installers" {
    ]
    resources = [aws_s3_bucket.software_installers.arn, "${aws_s3_bucket.software_installers.arn}/*"]
  }
+  dynamic "statement" {
+    for_each = local.software_installers_kms_policy
+    content {
+      sid       = try(statement.value.sid, "")
+      actions   = try(statement.value.actions, [])
+      resources = try(statement.value.resources, [])
+      effect    = try(statement.value.effect, null)
+      dynamic "principals" {
+        for_each = try(statement.value.principals, [])
+        content {
+          type        = principals.value.type
+          identifiers = principals.value.identifiers
+        }
+      }
+      dynamic "condition" {
+        for_each = try(statement.value.conditions, [])
+        content {
+          test     = condition.value.test
+          variable = condition.value.variable
+          values   = condition.value.values
+        }
+      }
+    }
+  }
 }

 resource "aws_iam_policy" "software_installers" {
@@ -26,7 +50,7 @@ resource "aws_iam_role_policy_attachment" "software_installers" {

 resource "aws_s3_bucket" "software_installers" { #tfsec:ignore:aws-s3-encryption-customer-key:exp:2022-07-01  #tfsec:ignore:aws-s3-enable-versioning #tfsec:ignore:aws-s3-enable-bucket-logging:exp:2022-06-15
  bucket_prefix = terraform.workspace
-  
+
  # Allow destroy of non-empty buckets
  force_destroy = true
 }
@@ -35,7 +59,8 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "software_installe
  bucket = aws_s3_bucket.software_installers.bucket
  rule {
    apply_server_side_encryption_by_default {
-      sse_algorithm = "aws:kms"
+      kms_master_key_id = aws_kms_key.software_installers.id
+      sse_algorithm     = "aws:kms"
    }
  }
 }
@@ -47,3 +72,12 @@ resource "aws_s3_bucket_public_access_block" "software_installers" {
  ignore_public_acls      = true
  restrict_public_buckets = true
 }
+
+resource "aws_kms_key" "software_installers" {
+  enable_key_rotation = true
+}
+
+resource "aws_kms_alias" "software_installers" {
+  target_key_id = aws_kms_key.software_installers.id
+  name          = "alias/${terraform.workspace}-software-installers"
+}
@@ -0,0 +1,24 @@
+module "cloudfront-software-installers" {
+  source            = "github.com/fleetdm/fleet-terraform/addons/cloudfront-software-installers?ref=tf-mod-addon-cloudfront-software-installers-v1.0.0"
+  customer          = terraform.workspace
+  s3_bucket         = aws_s3_bucket.software_installers.id
+  s3_kms_key_id     = aws_kms_key.software_installers.id
+  public_key        = data.aws_kms_secrets.cloudfront.plaintext["public_key"]
+  private_key       = data.aws_kms_secrets.cloudfront.plaintext["private_key"]
+  enable_logging    = true
+  logging_s3_bucket = module.logging_alb.log_s3_bucket_id
+}
+
+data "aws_kms_secrets" "cloudfront" {
+  secret {
+    name    = "public_key"
+    key_id  = aws_kms_key.customer_data_key.id
+    payload = file("${path.module}/resources/${terraform.workspace}/${local.cloudfront_key_basename}.pem.encrypted")
+  }
+  secret {
+    name    = "private_key"
+    key_id  = aws_kms_key.customer_data_key.id
+    payload = file("${path.module}/resources/${terraform.workspace}/${local.cloudfront_key_basename}.key.encrypted")
+  }
+}
+