diff --git a/.github/workflows/tfvalidate.yml b/.github/workflows/tfvalidate.yml index 209218ab47..d1316b4199 100644 --- a/.github/workflows/tfvalidate.yml +++ b/.github/workflows/tfvalidate.yml @@ -41,7 +41,7 @@ jobs: - name: Install terraform uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3 with: - terraform_version: 1.9.0 + terraform_version: 1.10.4 # If we want to test more of these, consider using a matrix. With a matrix of directories, all terraform modules could be fully tested and potentially in parallel. - name: Validate loadtesting working-directory: ./infrastructure/loadtesting/terraform diff --git a/infrastructure/loadtesting/terraform/ecs-iam.tf b/infrastructure/loadtesting/terraform/ecs-iam.tf index 6ae9cf9753..12e48aa5dc 100644 --- a/infrastructure/loadtesting/terraform/ecs-iam.tf +++ b/infrastructure/loadtesting/terraform/ecs-iam.tf @@ -81,3 +81,40 @@ resource "aws_iam_role_policy_attachment" "attachment" { policy_arn = aws_iam_policy.main.arn role = aws_iam_role.main.name } + + + +data "aws_iam_policy_document" "fleet-execution" { + // allow fleet application to obtain the database password from secrets manager + statement { + effect = "Allow" + actions = ["secretsmanager:GetSecretValue"] + resources = [ + aws_secretsmanager_secret.database_password_secret.arn, + aws_secretsmanager_secret.fleet_server_private_key.arn + ] + } +} + +resource "aws_iam_role_policy_attachment" "execution_extras" { + for_each = toset(local.extra_execution_iam_policies) + policy_arn = each.value + role = aws_iam_role.execution.name +} + +resource "aws_iam_policy" "execution" { + name = local.iam.execution.policy_name + description = "IAM policy that Fleet application uses to define access to AWS resources" + policy = data.aws_iam_policy_document.fleet-execution.json +} + +resource "aws_iam_role_policy_attachment" "execution" { + policy_arn = aws_iam_policy.execution.arn + role = aws_iam_role.execution.name +} + +resource "aws_iam_role" "execution" { + name = local.iam.execution.name + description = "The execution role for Fleet in ECS" + assume_role_policy = data.aws_iam_policy_document.assume_role.json +} \ No newline at end of file diff --git a/infrastructure/loadtesting/terraform/ecs.tf b/infrastructure/loadtesting/terraform/ecs.tf index fb1b13205a..834079cb39 100644 --- a/infrastructure/loadtesting/terraform/ecs.tf +++ b/infrastructure/loadtesting/terraform/ecs.tf @@ -113,7 +113,7 @@ resource "aws_ecs_task_definition" "backend" { awslogs-stream-prefix = "fleet" } }, - secrets = [ + secrets = concat([ { name = "FLEET_MYSQL_PASSWORD" valueFrom = aws_secretsmanager_secret.database_password_secret.arn @@ -130,7 +130,7 @@ resource "aws_ecs_task_definition" "backend" { name = "FLEET_SERVER_PRIVATE_KEY" valueFrom = aws_secretsmanager_secret.fleet_server_private_key.arn } - ] + ], local.secrets) environment = concat([ { name = "FLEET_LOGGING_JSON" diff --git a/infrastructure/loadtesting/terraform/firehose.tf b/infrastructure/loadtesting/terraform/firehose.tf index ac22ca1d6b..65235e7c3d 100644 --- a/infrastructure/loadtesting/terraform/firehose.tf +++ b/infrastructure/loadtesting/terraform/firehose.tf @@ -1,6 +1,6 @@ resource "aws_s3_bucket" "osquery-results" { #tfsec:ignore:aws-s3-encryption-customer-key tfsec:ignore:aws-s3-enable-bucket-logging tfsec:ignore:aws-s3-enable-versioning bucket = "${local.prefix}-loadtest-osquery-logs-archive" - + # Allow destroy of non-empty buckets force_destroy = true @@ -42,7 +42,7 @@ resource "aws_s3_bucket_public_access_block" "osquery-results" { resource "aws_s3_bucket" "osquery-status" { #tfsec:ignore:aws-s3-encryption-customer-key tfsec:ignore:aws-s3-enable-bucket-logging tfsec:ignore:aws-s3-enable-versioning bucket = "${local.prefix}-loadtest-osquery-status-archive" - + # Allow destroy of non-empty buckets force_destroy = true @@ -151,9 +151,9 @@ data "aws_iam_policy_document" "osquery_firehose_assume_role" { resource "aws_kinesis_firehose_delivery_stream" "osquery_results" { name = "${local.prefix}-osquery_results" - destination = "s3" + destination = "extended_s3" - s3_configuration { + extended_s3_configuration { role_arn = aws_iam_role.firehose-results.arn bucket_arn = aws_s3_bucket.osquery-results.arn } @@ -161,9 +161,9 @@ resource "aws_kinesis_firehose_delivery_stream" "osquery_results" { resource "aws_kinesis_firehose_delivery_stream" "osquery_status" { name = "${local.prefix}-osquery_status" - destination = "s3" + destination = "extended_s3" - s3_configuration { + extended_s3_configuration { role_arn = aws_iam_role.firehose-status.arn bucket_arn = aws_s3_bucket.osquery-status.arn } diff --git a/infrastructure/loadtesting/terraform/init.tf b/infrastructure/loadtesting/terraform/init.tf index 0ae274d925..27fe630893 100644 --- a/infrastructure/loadtesting/terraform/init.tf +++ b/infrastructure/loadtesting/terraform/init.tf @@ -14,7 +14,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.32.0" + version = "~> 5.0" } docker = { source = "kreuzwerker/docker" diff --git a/infrastructure/loadtesting/terraform/kms.tf b/infrastructure/loadtesting/terraform/kms.tf new file mode 100644 index 0000000000..d2b9c81043 --- /dev/null +++ b/infrastructure/loadtesting/terraform/kms.tf @@ -0,0 +1,12 @@ +resource "aws_kms_key" "customer_data_key" { + description = "key used to encrypt sensitive data stored in terraform" +} + +resource "aws_kms_alias" "alias" { + name = "alias/${terraform.workspace}-terraform-encrypted" + target_key_id = aws_kms_key.customer_data_key.id +} + +output "kms_key_id" { + value = aws_kms_key.customer_data_key.id +} \ No newline at end of file diff --git a/infrastructure/loadtesting/terraform/locals.tf b/infrastructure/loadtesting/terraform/locals.tf index 05a90bb692..edc3546d4e 100644 --- a/infrastructure/loadtesting/terraform/locals.tf +++ b/infrastructure/loadtesting/terraform/locals.tf @@ -1,6 +1,20 @@ locals { name = "fleetdm-${terraform.workspace}" prefix = "fleet-${terraform.workspace}" + extra_execution_iam_policies = concat( + # module.cloudfront-software-installers.extra_execution_iam_policies, + [] + ) + iam = { + role = { + name = "${terraform.workspace}-role" + policy_name = "${terraform.workspace}-iam-policy" + } + execution = { + name = "${terraform.workspace}-execution-role" + policy_name = "${terraform.workspace}-iam-policy-execution" + } + } additional_env_vars = [for k, v in merge({ "FLEET_VULNERABILITIES_DATABASES_PATH" : "/home/fleet" "FLEET_OSQUERY_ENABLE_ASYNC_HOST_PROCESSING" : "false" @@ -19,5 +33,24 @@ locals { "10.255.2.0/24", "10.255.3.0/24", ] - -} + software_installers_kms_policy = [{ + sid = "AllowSoftwareInstallersKMSAccess" + actions = [ + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:Encrypt*", + "kms:Describe*", + "kms:Decrypt*" + ] + resources = [aws_kms_key.software_installers.arn] + effect = "Allow" + }] + extra_secrets = merge( + # module.cloudfront-software-installers.extra_secrets + ) + secrets = [for k, v in local.extra_secrets : { + name = k + valueFrom = v + }] + cloudfront_key_basename = "cloudfront" +} \ No newline at end of file diff --git a/infrastructure/loadtesting/terraform/log-alb.tf b/infrastructure/loadtesting/terraform/log-alb.tf new file mode 100644 index 0000000000..eaeac53c6f --- /dev/null +++ b/infrastructure/loadtesting/terraform/log-alb.tf @@ -0,0 +1,5 @@ +module "logging_alb" { + source = "github.com/fleetdm/fleet-terraform/addons/logging-alb?depth=1&ref=tf-mod-addon-logging-alb-v1.3.0" + prefix = terraform.workspace + enable_athena = true +} \ No newline at end of file diff --git a/infrastructure/loadtesting/terraform/readme.md b/infrastructure/loadtesting/terraform/readme.md index 4dabd39d2e..64c139dcbb 100644 --- a/infrastructure/loadtesting/terraform/readme.md +++ b/infrastructure/loadtesting/terraform/readme.md @@ -74,6 +74,134 @@ This has an impact on real devices because they will not be notified of any comm - `-mdm_prob 1.0` - `-mdm_scep_challenge` set to the same value as `FLEET_MDM_APPLE_SCEP_CHALLENGE` above. +### Enabling Cloudfront + +> Do not commit your `BRANCH_NAME` if any files exist in `resources/TERRAFORM_WORKSPACE/` without a .encrypted extension. +> This step assumes that you've already successfully executed terraform apply and have a `kms_key_id` output. + +1. Under the loadtesting directory, create directory `resources/TERRAFORM_WORKSPACE/`. Your `TERRAFORM_WORKSPACE` value can be retrieved with `terraform workspace show`. + +2. Change directory to `resources/TERRAFORM_WORKSPACE/` + +3. Create your keys + +``` +openssl genrsa -out cloudfront.key 2048 +openssl rsa -pubout -in cloudfront.key -out cloudfront.pem +``` + +4. Create `encrypt.sh` (store the script in in the terraform directory, two directories up `../..`) + +``` +#!/bin/bash + +set -e + +function usage() { + cat <<-EOUSAGE + + Usage: $(basename ${0}) [AWS_PROFILE] + + This script encrypts an plaintext file from SOURCE into an + AWS KMS encrypted DESTINATION file. Optionally you + may provide the AWS_PROFILE you wish to use to run the aws kms + commands. + + EOUSAGE + exit 1 +} + +[ $# -lt 3 ] && usage + +if [ -n "${4}" ]; then + export AWS_PROFILE=${4} +fi + +aws kms encrypt --key-id "${1:?}" --plaintext fileb://<(cat "${2:?}") --output text --query CiphertextBlob > "${3:?}" +``` + +5. Encrypt the objects using `encrypt.sh` + +``` +for i in *; do ../../encrypt.sh $i $i.encrypted; done +for i in *.encrypted; do rm ${i/.encrypted/}; done +``` + +6. Change back to the terraform directory (two directories up ../..) + +7. If the name of your public/private cloudfront key is not `cloudfront.pem|.key`, update `locals.tf` + +``` +# Set the following variable value to the base name of your cloudfront key, no extension. +cloudfront_key_basename = cloudfront +``` + +8. Copy and make `cloudfront.tf` + +``` +cp template/cloudfront.tf.disabled cloudfront.tf +``` + +9. In `locals.tf` uncomment the following line, under `extra_secrets` + +``` +module.cloudfront-software-installers.extra_secrets, +``` + +Example: You should end up with something that looks like the following block + +``` + extra_secrets = merge( + module.cloudfront-software-installers.extra_secrets + ) +``` + +10. Initialize terraform and upgrade any necessary dependencies + +``` +terraform init -upgrade +``` + +11. Apply the terraform + +``` +terraform apply -var tag=BRANCH_NAME +``` + +12. In `locals.tf` uncomment the following line, under `extra_execution_iam_policies`. + +``` +module.cloudfront-software-installers.extra_execution_iam_policies, +``` + +You should end up with something that looks like this. + +``` + extra_execution_iam_policies = concat( + module.cloudfront-software-installers.extra_execution_iam_policies, + [] + ) +``` + +13. Apply the terraform + +``` +terraform apply -var tag=BRANCH_NAME +``` + +14. Cloudfront should now be enabled. + +15. If you had previously uploaded any software installers, they need to be re-encrypted by finding and targeting your bucket with the following commands. + +``` +# List buckets matching your BRANCH_NAME +aws s3 ls | grep BRANCH_NAME + +# Replace with the software_instalelrs bucket name. +aws s3 cp s3:/// s3:/// --recursive +``` + + ### Running a loadtest We run simulated hosts in containers of 500 at a time. Once the infrastructure is running, you can run the following command: diff --git a/infrastructure/loadtesting/terraform/redis.tf b/infrastructure/loadtesting/terraform/redis.tf index 0eedba51c8..54143f4f86 100644 --- a/infrastructure/loadtesting/terraform/redis.tf +++ b/infrastructure/loadtesting/terraform/redis.tf @@ -1,20 +1,20 @@ resource "aws_elasticache_replication_group" "default" { - availability_zones = ["us-east-2a", "us-east-2b", "us-east-2c"] - engine = "redis" - parameter_group_name = aws_elasticache_parameter_group.default.id - subnet_group_name = data.terraform_remote_state.shared.outputs.vpc.elasticache_subnet_group_name - security_group_ids = [aws_security_group.redis.id, aws_security_group.backend.id] - replication_group_id = "${local.prefix}-redis" - num_cache_clusters = 3 - node_type = var.redis_instance_type - engine_version = "6.2" - port = "6379" - snapshot_retention_limit = 0 - automatic_failover_enabled = true - at_rest_encryption_enabled = false #tfsec:ignore:aws-elasticache-enable-at-rest-encryption - transit_encryption_enabled = false #tfsec:ignore:aws-elasticache-enable-in-transit-encryption - apply_immediately = true - description = "${local.prefix}-redis" + preferred_cache_cluster_azs = ["us-east-2a", "us-east-2b", "us-east-2c"] + engine = "redis" + parameter_group_name = aws_elasticache_parameter_group.default.id + subnet_group_name = data.terraform_remote_state.shared.outputs.vpc.elasticache_subnet_group_name + security_group_ids = [aws_security_group.redis.id, aws_security_group.backend.id] + replication_group_id = "${local.prefix}-redis" + num_cache_clusters = 3 + node_type = var.redis_instance_type + engine_version = "6.2" + port = "6379" + snapshot_retention_limit = 0 + automatic_failover_enabled = true + at_rest_encryption_enabled = false #tfsec:ignore:aws-elasticache-enable-at-rest-encryption + transit_encryption_enabled = false #tfsec:ignore:aws-elasticache-enable-in-transit-encryption + apply_immediately = true + description = "${local.prefix}-redis" } diff --git a/infrastructure/loadtesting/terraform/s3.tf b/infrastructure/loadtesting/terraform/s3.tf index 1016dab37c..5778a29743 100644 --- a/infrastructure/loadtesting/terraform/s3.tf +++ b/infrastructure/loadtesting/terraform/s3.tf @@ -13,6 +13,30 @@ data "aws_iam_policy_document" "software_installers" { ] resources = [aws_s3_bucket.software_installers.arn, "${aws_s3_bucket.software_installers.arn}/*"] } + dynamic "statement" { + for_each = local.software_installers_kms_policy + content { + sid = try(statement.value.sid, "") + actions = try(statement.value.actions, []) + resources = try(statement.value.resources, []) + effect = try(statement.value.effect, null) + dynamic "principals" { + for_each = try(statement.value.principals, []) + content { + type = principals.value.type + identifiers = principals.value.identifiers + } + } + dynamic "condition" { + for_each = try(statement.value.conditions, []) + content { + test = condition.value.test + variable = condition.value.variable + values = condition.value.values + } + } + } + } } resource "aws_iam_policy" "software_installers" { @@ -26,7 +50,7 @@ resource "aws_iam_role_policy_attachment" "software_installers" { resource "aws_s3_bucket" "software_installers" { #tfsec:ignore:aws-s3-encryption-customer-key:exp:2022-07-01 #tfsec:ignore:aws-s3-enable-versioning #tfsec:ignore:aws-s3-enable-bucket-logging:exp:2022-06-15 bucket_prefix = terraform.workspace - + # Allow destroy of non-empty buckets force_destroy = true } @@ -35,7 +59,8 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "software_installe bucket = aws_s3_bucket.software_installers.bucket rule { apply_server_side_encryption_by_default { - sse_algorithm = "aws:kms" + kms_master_key_id = aws_kms_key.software_installers.id + sse_algorithm = "aws:kms" } } } @@ -47,3 +72,12 @@ resource "aws_s3_bucket_public_access_block" "software_installers" { ignore_public_acls = true restrict_public_buckets = true } + +resource "aws_kms_key" "software_installers" { + enable_key_rotation = true +} + +resource "aws_kms_alias" "software_installers" { + target_key_id = aws_kms_key.software_installers.id + name = "alias/${terraform.workspace}-software-installers" +} \ No newline at end of file diff --git a/infrastructure/loadtesting/terraform/template/cloudfront.tf.disabled b/infrastructure/loadtesting/terraform/template/cloudfront.tf.disabled new file mode 100644 index 0000000000..541a283f43 --- /dev/null +++ b/infrastructure/loadtesting/terraform/template/cloudfront.tf.disabled @@ -0,0 +1,24 @@ +module "cloudfront-software-installers" { + source = "github.com/fleetdm/fleet-terraform/addons/cloudfront-software-installers?ref=tf-mod-addon-cloudfront-software-installers-v1.0.0" + customer = terraform.workspace + s3_bucket = aws_s3_bucket.software_installers.id + s3_kms_key_id = aws_kms_key.software_installers.id + public_key = data.aws_kms_secrets.cloudfront.plaintext["public_key"] + private_key = data.aws_kms_secrets.cloudfront.plaintext["private_key"] + enable_logging = true + logging_s3_bucket = module.logging_alb.log_s3_bucket_id +} + +data "aws_kms_secrets" "cloudfront" { + secret { + name = "public_key" + key_id = aws_kms_key.customer_data_key.id + payload = file("${path.module}/resources/${terraform.workspace}/${local.cloudfront_key_basename}.pem.encrypted") + } + secret { + name = "private_key" + key_id = aws_kms_key.customer_data_key.id + payload = file("${path.module}/resources/${terraform.workspace}/${local.cloudfront_key_basename}.key.encrypted") + } +} +