From 69c0434e88e2f3ce6dbaecca151d7f60296844bc Mon Sep 17 00:00:00 2001 From: Zachary Winnerman <98712682+zwinnerman-fleetdm@users.noreply.github.com> Date: Tue, 15 Feb 2022 13:00:24 -0500 Subject: [PATCH] Add terraform for the loadtesting environment (#4172) * Add terraform for the loadtesting environment * Add some checkov stuff and minor updates * Remove defunct commented stuff * Add separate cloudwatch namespace for migrations * Remove defunct code * checkin --- .../loadtesting/terraform/.terraform-version | 1 + tools/loadtesting/terraform/docker/Dockerfile | 2 + tools/loadtesting/terraform/ecr.tf | 51 +++ tools/loadtesting/terraform/ecs-iam.tf | 83 ++++ tools/loadtesting/terraform/ecs-sgs.tf | 112 +++++ tools/loadtesting/terraform/ecs.tf | 398 ++++++++++++++++++ tools/loadtesting/terraform/elasticsearch.sh | 101 +++++ tools/loadtesting/terraform/elasticsearch.tf | 178 ++++++++ tools/loadtesting/terraform/firehose.tf | 132 ++++++ tools/loadtesting/terraform/locals.tf | 17 + tools/loadtesting/terraform/main.tf | 44 ++ tools/loadtesting/terraform/outputs.tf | 15 + tools/loadtesting/terraform/r53.tf | 62 +++ tools/loadtesting/terraform/rds.tf | 77 ++++ tools/loadtesting/terraform/readme.md | 12 + tools/loadtesting/terraform/redis.tf | 59 +++ tools/loadtesting/terraform/state/state.yml | 13 + tools/loadtesting/terraform/variables.tf | 9 + tools/loadtesting/terraform/vpc.tf | 25 ++ 19 files changed, 1391 insertions(+) create mode 100644 tools/loadtesting/terraform/.terraform-version create mode 100644 tools/loadtesting/terraform/docker/Dockerfile create mode 100644 tools/loadtesting/terraform/ecr.tf create mode 100644 tools/loadtesting/terraform/ecs-iam.tf create mode 100644 tools/loadtesting/terraform/ecs-sgs.tf create mode 100644 tools/loadtesting/terraform/ecs.tf create mode 100644 tools/loadtesting/terraform/elasticsearch.sh create mode 100644 tools/loadtesting/terraform/elasticsearch.tf create mode 100644 tools/loadtesting/terraform/firehose.tf create mode 100644 tools/loadtesting/terraform/locals.tf create mode 100644 tools/loadtesting/terraform/main.tf create mode 100644 tools/loadtesting/terraform/outputs.tf create mode 100644 tools/loadtesting/terraform/r53.tf create mode 100644 tools/loadtesting/terraform/rds.tf create mode 100644 tools/loadtesting/terraform/readme.md create mode 100644 tools/loadtesting/terraform/redis.tf create mode 100644 tools/loadtesting/terraform/state/state.yml create mode 100644 tools/loadtesting/terraform/variables.tf create mode 100644 tools/loadtesting/terraform/vpc.tf diff --git a/tools/loadtesting/terraform/.terraform-version b/tools/loadtesting/terraform/.terraform-version new file mode 100644 index 0000000000..a6a3a43c3a --- /dev/null +++ b/tools/loadtesting/terraform/.terraform-version @@ -0,0 +1 @@ +1.0.4 \ No newline at end of file diff --git a/tools/loadtesting/terraform/docker/Dockerfile b/tools/loadtesting/terraform/docker/Dockerfile new file mode 100644 index 0000000000..fe63778275 --- /dev/null +++ b/tools/loadtesting/terraform/docker/Dockerfile @@ -0,0 +1,2 @@ +ARG TAG +FROM fleetdm/fleet:$TAG diff --git a/tools/loadtesting/terraform/ecr.tf b/tools/loadtesting/terraform/ecr.tf new file mode 100644 index 0000000000..447ff42a71 --- /dev/null +++ b/tools/loadtesting/terraform/ecr.tf @@ -0,0 +1,51 @@ +resource "aws_kms_key" "main" { + description = "${local.prefix}-${random_pet.db_secret_postfix.id}" + deletion_window_in_days = 10 + enable_key_rotation = true +} + +resource "aws_ecr_repository" "prometheus-to-cloudwatch" { + name = "prometheus-to-cloudwatch" + image_tag_mutability = "IMMUTABLE" + + image_scanning_configuration { + scan_on_push = true + } + + encryption_configuration { + encryption_type = "KMS" + kms_key = aws_kms_key.main.arn + } +} + +resource "aws_ecr_repository" "fleet" { + name = "fleet" + image_tag_mutability = "IMMUTABLE" + + image_scanning_configuration { + scan_on_push = true + } + + encryption_configuration { + encryption_type = "KMS" + kms_key = aws_kms_key.main.arn + } +} + +data "aws_ecr_authorization_token" "token" {} + +resource "docker_registry_image" "fleet" { + name = "${aws_ecr_repository.fleet.repository_url}:${var.tag}-${split(":", data.docker_registry_image.dockerhub.sha256_digest)[1]}" + + build { + context = "${path.cwd}/docker/" + build_args = { + TAG = var.tag + } + pull_parent = true + } +} + +data "docker_registry_image" "dockerhub" { + name = "fleetdm/fleet:${var.tag}" +} diff --git a/tools/loadtesting/terraform/ecs-iam.tf b/tools/loadtesting/terraform/ecs-iam.tf new file mode 100644 index 0000000000..180e1ef22f --- /dev/null +++ b/tools/loadtesting/terraform/ecs-iam.tf @@ -0,0 +1,83 @@ +data "aws_iam_policy_document" "fleet" { + statement { + effect = "Allow" + actions = ["cloudwatch:PutMetricData"] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = ["secretsmanager:GetSecretValue"] + resources = [aws_secretsmanager_secret.database_password_secret.arn, data.aws_secretsmanager_secret.license.arn] + } + + // useful when there is a static number of mysql cluster members + dynamic "statement" { + for_each = module.aurora_mysql.rds_cluster_instance_dbi_resource_ids + content { + effect = "Allow" + actions = ["rds-db:connect"] + resources = ["arn:aws:rds-db:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:dbuser:${statement.value}/${module.aurora_mysql.rds_cluster_master_username}"] + } + } + + // allow access to any database via IAM that has the var.database_user user + // useful when you are autoscaling mysql read replicas dynamically + statement { + effect = "Allow" + actions = ["rds-db:connect"] + resources = ["arn:aws:rds-db:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:dbuser:*/${module.aurora_mysql.rds_cluster_master_username}"] + } + + statement { + effect = "Allow" + actions = [ + "firehose:DescribeDeliveryStream", + "firehose:PutRecord", + "firehose:PutRecordBatch", + ] + resources = [aws_kinesis_firehose_delivery_stream.osquery_results.arn, aws_kinesis_firehose_delivery_stream.osquery_status.arn] + } + + statement { + actions = [ + "kms:Encrypt*", + "kms:Decrypt*", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:Describe*" + ] + resources = [aws_kms_key.main.arn] + } +} + +data "aws_iam_policy_document" "assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + identifiers = ["ecs.amazonaws.com", "ecs-tasks.amazonaws.com"] + type = "Service" + } + } +} + +resource "aws_iam_role" "main" { + name = "fleetdm-role" + assume_role_policy = data.aws_iam_policy_document.assume_role.json +} + +resource "aws_iam_role_policy_attachment" "role_attachment" { + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + role = aws_iam_role.main.name +} + +resource "aws_iam_policy" "main" { + name = "fleet-iam-policy" + policy = data.aws_iam_policy_document.fleet.json +} + +resource "aws_iam_role_policy_attachment" "attachment" { + policy_arn = aws_iam_policy.main.arn + role = aws_iam_role.main.name +} diff --git a/tools/loadtesting/terraform/ecs-sgs.tf b/tools/loadtesting/terraform/ecs-sgs.tf new file mode 100644 index 0000000000..87d3a0d22e --- /dev/null +++ b/tools/loadtesting/terraform/ecs-sgs.tf @@ -0,0 +1,112 @@ +# Security group for the public internet facing load balancer +resource "aws_security_group" "lb" { + name = "${local.prefix} load balancer" + description = "${local.prefix} Load balancer security group" + vpc_id = module.vpc.vpc_id +} + +# Allow traffic from public internet +resource "aws_security_group_rule" "lb-ingress" { + description = "${local.prefix}: allow traffic from public internet" + type = "ingress" + + from_port = "443" + to_port = "443" + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.lb.id +} + +resource "aws_security_group_rule" "lb-http-ingress" { + description = "${local.prefix}: allow traffic from public internet" + type = "ingress" + + from_port = "80" + to_port = "80" + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.lb.id +} + +resource "aws_security_group_rule" "lb-es" { + description = "${local.prefix}: allow traffic from public internet" + type = "ingress" + + from_port = "9200" + to_port = "9200" + protocol = "tcp" + cidr_blocks = ["10.0.0.0/8"] + + security_group_id = aws_security_group.lb.id +} +resource "aws_security_group_rule" "lb-es-apm" { + description = "${local.prefix}: allow traffic from public internet" + type = "ingress" + + from_port = "8200" + to_port = "8200" + protocol = "tcp" + cidr_blocks = ["10.0.0.0/8"] + + security_group_id = aws_security_group.lb.id +} +resource "aws_security_group_rule" "lb-kibana" { + description = "${local.prefix}: allow traffic from public internet" + type = "ingress" + + from_port = "5601" + to_port = "5601" + protocol = "tcp" + cidr_blocks = ["10.0.0.0/8"] + + security_group_id = aws_security_group.lb.id +} + +# Allow outbound traffic +resource "aws_security_group_rule" "lb-egress" { + description = "${local.prefix}: allow all outbound traffic" + type = "egress" + + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.lb.id +} + +# Security group for the backends that run the application. +# Allows traffic from the load balancer +resource "aws_security_group" "backend" { + name = "${local.prefix} backend" + description = "${local.prefix} Backend security group" + vpc_id = module.vpc.vpc_id + +} + +# Allow traffic from the load balancer to the backends +resource "aws_security_group_rule" "backend-ingress" { + description = "${local.prefix}: allow traffic from load balancer" + type = "ingress" + + from_port = "8080" + to_port = "8080" + protocol = "tcp" + source_security_group_id = aws_security_group.lb.id + security_group_id = aws_security_group.backend.id +} + +# Allow outbound traffic from the backends +resource "aws_security_group_rule" "backend-egress" { + description = "${local.prefix}: allow all outbound traffic" + type = "egress" + + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + + security_group_id = aws_security_group.backend.id +} diff --git a/tools/loadtesting/terraform/ecs.tf b/tools/loadtesting/terraform/ecs.tf new file mode 100644 index 0000000000..0e45cc329d --- /dev/null +++ b/tools/loadtesting/terraform/ecs.tf @@ -0,0 +1,398 @@ +resource "aws_alb" "main" { + name = "fleetdm" + internal = false + security_groups = [aws_security_group.lb.id, aws_security_group.backend.id] + subnets = module.vpc.public_subnets + idle_timeout = 600 + drop_invalid_header_fields = true + #checkov:skip=CKV_AWS_150:don't like it +} + +resource "aws_alb_target_group" "main" { + name = "fleetdm" + protocol = "HTTP" + target_type = "ip" + port = "8080" + vpc_id = module.vpc.vpc_id + deregistration_delay = 30 + + load_balancing_algorithm_type = "least_outstanding_requests" + + health_check { + path = "/healthz" + matcher = "200" + timeout = 10 + interval = 15 + healthy_threshold = 5 + unhealthy_threshold = 5 + } + + depends_on = [aws_alb.main] +} + +resource "aws_alb_listener" "https-fleetdm" { + load_balancer_arn = aws_alb.main.arn + port = 443 + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-FS-1-2-Res-2019-08" + certificate_arn = aws_acm_certificate_validation.dogfood_fleetdm_com.certificate_arn + + default_action { + target_group_arn = aws_alb_target_group.main.arn + type = "forward" + } +} + +resource "aws_alb_listener" "http" { + load_balancer_arn = aws_alb.main.arn + port = "80" + protocol = "HTTP" + + default_action { + type = "redirect" + + redirect { + port = "443" + protocol = "HTTPS" + status_code = "HTTP_301" + } + } +} + +resource "aws_ecs_cluster" "fleet" { + name = "${local.prefix}-backend" + + setting { + name = "containerInsights" + value = "enabled" + } +} + +resource "aws_ecs_service" "fleet" { + name = "fleet" + launch_type = "FARGATE" + cluster = aws_ecs_cluster.fleet.id + task_definition = aws_ecs_task_definition.backend.arn + desired_count = 10 + deployment_minimum_healthy_percent = 100 + deployment_maximum_percent = 200 + health_check_grace_period_seconds = 30 + + load_balancer { + target_group_arn = aws_alb_target_group.main.arn + container_name = "fleet" + container_port = 8080 + } + + network_configuration { + subnets = module.vpc.private_subnets + security_groups = [aws_security_group.backend.id] + } + + depends_on = [aws_alb_listener.http, aws_alb_listener.https-fleetdm] +} + +resource "aws_cloudwatch_log_group" "backend" { + name = "fleetdm" + retention_in_days = 1 +} + +data "aws_region" "current" {} + +data "aws_secretsmanager_secret" "license" { + name = "/fleet/license" +} + +resource "aws_ecs_task_definition" "backend" { + family = "fleet" + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + execution_role_arn = aws_iam_role.main.arn + task_role_arn = aws_iam_role.main.arn + cpu = 1024 + memory = 2048 + container_definitions = jsonencode( + [ + { + name = "prometheus-exporter" + image = "917007347864.dkr.ecr.us-east-2.amazonaws.com/prometheus-to-cloudwatch:latest" + essential = false + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.backend.name + awslogs-region = data.aws_region.current.name + awslogs-stream-prefix = "fleet-prometheus-exporter" + } + } + environment = [ + { + name = "CLOUDWATCH_NAMESPACE" + value = "fleet-loadtest" + }, + { + name = "CLOUDWATCH_REGION" + value = "us-east-2" + }, + { + name = "PROMETHEUS_SCRAPE_URL" + value = "http://localhost:8080/metrics" + }, + ], + }, + { + name = "fleet" + image = docker_registry_image.fleet.name + cpu = 1024 + memory = 2048 + mountPoints = [] + volumesFrom = [] + essential = true + portMappings = [ + { + # This port is the same that the contained application also uses + containerPort = 8080 + hostPort = 8080 + protocol = "tcp" + } + ] + ulimits = [ + { + softLimit = 9999, + hardLimit = 9999, + name = "nofile" + } + ] + networkMode = "awsvpc" + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.backend.name + awslogs-region = data.aws_region.current.name + awslogs-stream-prefix = "fleet" + } + }, + secrets = [ + { + name = "FLEET_MYSQL_PASSWORD" + valueFrom = aws_secretsmanager_secret.database_password_secret.arn + }, + { + name = "FLEET_MYSQL_READ_REPLICA_PASSWORD" + valueFrom = aws_secretsmanager_secret.database_password_secret.arn + }, + { + name = "FLEET_LICENSE_KEY" + valueFrom = data.aws_secretsmanager_secret.license.arn + } + ] + environment = concat([ + { + name = "FLEET_MYSQL_USERNAME" + value = module.aurora_mysql.rds_cluster_master_username + }, + { + name = "FLEET_MYSQL_DATABASE" + value = module.aurora_mysql.rds_cluster_database_name + }, + { + name = "FLEET_MYSQL_ADDRESS" + value = "${module.aurora_mysql.rds_cluster_endpoint}:3306" + }, + { + name = "FLEET_MYSQL_READ_REPLICA_USERNAME" + value = module.aurora_mysql.rds_cluster_master_username + }, + { + name = "FLEET_MYSQL_READ_REPLICA_DATABASE" + value = module.aurora_mysql.rds_cluster_database_name + }, + { + name = "FLEET_MYSQL_READ_REPLICA_ADDRESS" + value = "${module.aurora_mysql.rds_cluster_reader_endpoint}:3306" + }, + { + name = "FLEET_REDIS_ADDRESS" + value = "${aws_elasticache_replication_group.default.primary_endpoint_address}:6379" + }, + { + name = "FLEET_REDIS_CLUSTER_FOLLOW_REDIRECTIONS" + value = "true" + }, + { + name = "FLEET_FIREHOSE_STATUS_STREAM" + value = aws_kinesis_firehose_delivery_stream.osquery_status.name + }, + { + name = "FLEET_FIREHOSE_RESULT_STREAM" + value = aws_kinesis_firehose_delivery_stream.osquery_results.name + }, + { + name = "FLEET_FIREHOSE_REGION" + value = data.aws_region.current.name + }, + { + name = "FLEET_OSQUERY_STATUS_LOG_PLUGIN" + value = "firehose" + }, + { + name = "FLEET_OSQUERY_RESULT_LOG_PLUGIN" + value = "firehose" + }, + { + name = "FLEET_SERVER_TLS" + value = "false" + }, + { + name = "FLEET_REDIS_MAX_IDLE_CONNS" + value = "100" + }, + { + name = "FLEET_REDIS_MAX_OPEN_CONNS" + value = "100" + }, + { + name = "FLEET_OSQUERY_ASYNC_HOST_REDIS_SCAN_KEYS_COUNT" + value = "10000" + } + ], local.additional_env_vars) + } + ]) + lifecycle { + create_before_destroy = true + } +} + + +resource "aws_ecs_task_definition" "migration" { + family = "fleet-migrate" + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + execution_role_arn = aws_iam_role.main.arn + task_role_arn = aws_iam_role.main.arn + cpu = 1024 + memory = 2048 + container_definitions = jsonencode( + [ + { + name = "fleet-prepare-db" + image = docker_registry_image.fleet.name + cpu = 1024 + memory = 2048 + mountPoints = [] + volumesFrom = [] + essential = true + portMappings = [ + { + # This port is the same that the contained application also uses + containerPort = 8080 + protocol = "tcp" + } + ] + networkMode = "awsvpc" + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.backend.name + awslogs-region = data.aws_region.current.name + awslogs-stream-prefix = "fleet-migration" + } + }, + command = ["fleet", "prepare", "--no-prompt=true", "db"] + secrets = [ + { + name = "FLEET_MYSQL_PASSWORD" + valueFrom = aws_secretsmanager_secret.database_password_secret.arn + } + ] + environment = [ + { + name = "CLOUDWATCH_NAMESPACE" + value = "fleet-loadtest-migration" + }, + { + name = "CLOUDWATCH_REGION" + value = "us-east-2" + }, + { + name = "FLEET_MYSQL_USERNAME" + value = module.aurora_mysql.rds_cluster_master_username + }, + { + name = "FLEET_MYSQL_DATABASE" + value = module.aurora_mysql.rds_cluster_database_name + }, + { + name = "FLEET_MYSQL_ADDRESS" + value = "${module.aurora_mysql.rds_cluster_endpoint}:3306" + }, + { + name = "FLEET_REDIS_ADDRESS" + value = "${aws_elasticache_replication_group.default.primary_endpoint_address}:6379" + }, + ] + } + ]) + lifecycle { + create_before_destroy = true + } +} + +resource "aws_appautoscaling_target" "ecs_target" { + max_capacity = 100 + min_capacity = 10 + resource_id = "service/${aws_ecs_cluster.fleet.name}/${aws_ecs_service.fleet.name}" + scalable_dimension = "ecs:service:DesiredCount" + service_namespace = "ecs" +} + +resource "aws_appautoscaling_policy" "ecs_policy_memory" { + name = "fleet-memory-autoscaling" + policy_type = "TargetTrackingScaling" + resource_id = aws_appautoscaling_target.ecs_target.resource_id + scalable_dimension = aws_appautoscaling_target.ecs_target.scalable_dimension + service_namespace = aws_appautoscaling_target.ecs_target.service_namespace + + target_tracking_scaling_policy_configuration { + predefined_metric_specification { + predefined_metric_type = "ECSServiceAverageMemoryUtilization" + } + target_value = 80 + } +} + +resource "aws_appautoscaling_policy" "ecs_policy_cpu" { + name = "fleet-cpu-autoscaling" + policy_type = "TargetTrackingScaling" + resource_id = aws_appautoscaling_target.ecs_target.resource_id + scalable_dimension = aws_appautoscaling_target.ecs_target.scalable_dimension + service_namespace = aws_appautoscaling_target.ecs_target.service_namespace + + target_tracking_scaling_policy_configuration { + predefined_metric_specification { + predefined_metric_type = "ECSServiceAverageCPUUtilization" + } + + target_value = 60 + } +} + +output "fleet_migration_revision" { + value = aws_ecs_task_definition.migration.revision +} + +output "fleet_migration_subnets" { + value = jsonencode(aws_ecs_service.fleet.network_configuration[0].subnets) +} + +output "fleet_migration_security_groups" { + value = jsonencode(aws_ecs_service.fleet.network_configuration[0].security_groups) +} + +output "fleet_ecs_cluster_arn" { + value = aws_ecs_cluster.fleet.arn +} + +output "fleet_ecs_cluster_id" { + value = aws_ecs_cluster.fleet.id +} diff --git a/tools/loadtesting/terraform/elasticsearch.sh b/tools/loadtesting/terraform/elasticsearch.sh new file mode 100644 index 0000000000..dd0ef6eae0 --- /dev/null +++ b/tools/loadtesting/terraform/elasticsearch.sh @@ -0,0 +1,101 @@ +#!/bin/bash +#yum update -y +#yum install -y python3-pip git +#pip3 install ansible + +TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"` +REPO=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/ansible_repository` +PLAYBOOK_PATH=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/ansible_playbook_path` +PLAYBOOK_FILE=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/ansible_playbook_file` +BRANCH=`curl -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/ansible_branch` + +#git clone "${REPO}" ansible +#cd ansible +#git checkout "${BRANCH}" +#cd "${PLAYBOOK_PATH}" +#ansible-playbook -c local "${PLAYBOOK_FILE}" + +yum install -y docker +systemctl start docker.service +systemctl enable docker.service +curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/bin/docker-compose +chmod +x /usr/bin/docker-compose +cat << EOT >docker-compose.yml +version: '2.2' +services: + apm-server: + image: docker.elastic.co/apm/apm-server:7.15.2 + depends_on: + elasticsearch: + condition: service_healthy + kibana: + condition: service_healthy + cap_add: ["CHOWN", "DAC_OVERRIDE", "SETGID", "SETUID"] + cap_drop: ["ALL"] + ports: + - 8200:8200 + networks: + - elastic + command: > + apm-server -e + -E apm-server.rum.enabled=true + -E setup.kibana.host=kibana:5601 + -E setup.template.settings.index.number_of_replicas=0 + -E apm-server.kibana.enabled=true + -E apm-server.kibana.host=kibana:5601 + -E output.elasticsearch.hosts=["elasticsearch:9200"] + healthcheck: + interval: 10s + retries: 12 + test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:8200/ + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.15.2 + environment: + - bootstrap.memory_lock=true + - cluster.name=docker-cluster + - cluster.routing.allocation.disk.threshold_enabled=false + - discovery.type=single-node + - ES_JAVA_OPTS=-XX:UseAVX=2 -Xms1g -Xmx1g + ulimits: + memlock: + hard: -1 + soft: -1 + volumes: + - esdata:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - elastic + healthcheck: + interval: 20s + retries: 10 + test: curl -s http://localhost:9200/_cluster/health | grep -vq '"status":"red"' + + kibana: + image: docker.elastic.co/kibana/kibana:7.15.2 + depends_on: + elasticsearch: + condition: service_healthy + environment: + ELASTICSEARCH_URL: http://elasticsearch:9200 + ELASTICSEARCH_HOSTS: http://elasticsearch:9200 + ports: + - 5601:5601 + networks: + - elastic + healthcheck: + interval: 10s + retries: 20 + test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:5601/api/status + +volumes: + esdata: + driver: local + +networks: + elastic: + driver: bridge +EOT + +docker-compose up -d diff --git a/tools/loadtesting/terraform/elasticsearch.tf b/tools/loadtesting/terraform/elasticsearch.tf new file mode 100644 index 0000000000..2c29158504 --- /dev/null +++ b/tools/loadtesting/terraform/elasticsearch.tf @@ -0,0 +1,178 @@ +data "aws_default_tags" "current" {} + +resource "aws_autoscaling_group" "elasticstack" { + name = "${local.prefix}-elasticstack" + max_size = 1 + min_size = 1 + health_check_grace_period = 3000 + health_check_type = "ELB" + desired_capacity = 1 + force_delete = true + vpc_zone_identifier = module.vpc.private_subnets + target_group_arns = [aws_lb_target_group.elasticsearch.arn, aws_lb_target_group.elasticapm.arn, aws_lb_target_group.kibana.arn] + + launch_template { + id = aws_launch_template.elasticstack.id + version = "$Latest" + } + + timeouts { + delete = "15m" + } + + dynamic "tag" { + for_each = data.aws_default_tags.current.tags + content { + key = tag.key + value = tag.value + propagate_at_launch = true + } + } + + tag { + key = "ansible_repository" + value = "https://github.com/fleetdm/fleet.git" + propagate_at_launch = true + } + + tag { + key = "ansible_playbook_path" + value = "tools/loadtesting/terraform/elasticsearch_ansible" + propagate_at_launch = true + } + + tag { + key = "ansible_playbook_file" + value = "elasticsearch.yml" + propagate_at_launch = true + } + + tag { + key = "ansible_branch" + value = "zwinnerman-add-loadtest-infra-mine-fixup" + propagate_at_launch = true + } +} + +data "aws_iam_policy_document" "elasticstack" { + statement { + effect = "Allow" + actions = ["secretsmanager:GetSecretValue"] + resources = ["arn:aws:secretsmanager:us-east-2:917007347864:secret:/fleet/ssh/keys-7iQNe1"] + } +} + +data "aws_iam_policy_document" "assume_role_es" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + identifiers = ["ec2.amazonaws.com"] + type = "Service" + } + } +} + +resource "aws_iam_role" "elasticstack" { + name = "fleetdm-es-role" + assume_role_policy = data.aws_iam_policy_document.assume_role_es.json +} + +resource "aws_iam_role_policy_attachment" "role_attachment_es" { + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + role = aws_iam_role.elasticstack.name +} + +resource "aws_iam_policy" "elasticstack" { + name = "fleet-es-iam-policy" + policy = data.aws_iam_policy_document.elasticstack.json +} + +resource "aws_iam_role_policy_attachment" "elasticstack" { + policy_arn = aws_iam_policy.elasticstack.arn + role = aws_iam_role.elasticstack.name +} + +data "aws_ami" "amazonlinux" { + owners = ["amazon"] + most_recent = true + + filter { + name = "name" + values = ["amzn2-ami-hvm-*-x86_64-ebs"] + } +} + +resource "aws_launch_template" "elasticstack" { + name_prefix = "${local.prefix}-elasticstack" + image_id = data.aws_ami.amazonlinux.image_id + instance_type = "t3.large" + key_name = "zwinnerman" + vpc_security_group_ids = [aws_security_group.lb.id] + metadata_options { + instance_metadata_tags = "enabled" + http_endpoint = "enabled" + http_tokens = "required" + } + user_data = filebase64("${path.module}/elasticsearch.sh") +} + +resource "aws_alb_listener" "elasticsearch" { + load_balancer_arn = aws_alb.main.arn + port = 9200 + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-FS-1-2-Res-2019-08" + certificate_arn = aws_acm_certificate_validation.dogfood_fleetdm_com.certificate_arn + + default_action { + target_group_arn = aws_lb_target_group.elasticsearch.arn + type = "forward" + } +} + +resource "aws_lb_target_group" "elasticsearch" { + name = "${local.prefix}-elasticsearch" + port = 9200 + protocol = "HTTP" + vpc_id = module.vpc.vpc_id +} + +resource "aws_alb_listener" "elasticapm" { + load_balancer_arn = aws_alb.main.arn + port = 8200 + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-FS-1-2-Res-2019-08" + certificate_arn = aws_acm_certificate_validation.dogfood_fleetdm_com.certificate_arn + + default_action { + target_group_arn = aws_lb_target_group.elasticapm.arn + type = "forward" + } +} + +resource "aws_lb_target_group" "elasticapm" { + name = "${local.prefix}-elasticapm" + port = 8200 + protocol = "HTTP" + vpc_id = module.vpc.vpc_id +} + +resource "aws_alb_listener" "kibana" { + load_balancer_arn = aws_alb.main.arn + port = 5601 + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-FS-1-2-Res-2019-08" + certificate_arn = aws_acm_certificate_validation.dogfood_fleetdm_com.certificate_arn + + default_action { + target_group_arn = aws_lb_target_group.kibana.arn + type = "forward" + } +} + +resource "aws_lb_target_group" "kibana" { + name = "${local.prefix}-kibana" + port = 5601 + protocol = "HTTP" + vpc_id = module.vpc.vpc_id +} diff --git a/tools/loadtesting/terraform/firehose.tf b/tools/loadtesting/terraform/firehose.tf new file mode 100644 index 0000000000..73a6b87dcf --- /dev/null +++ b/tools/loadtesting/terraform/firehose.tf @@ -0,0 +1,132 @@ +resource "aws_s3_bucket" "osquery-results" { + bucket = "fleet-loadtest-osquery-logs-archive" + acl = "private" + + lifecycle_rule { + enabled = true + expiration { + days = 1 + } + } + + server_side_encryption_configuration { + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "aws:kms" + } + } + } + #checkov:skip=CKV_AWS_18:dev env + #checkov:skip=CKV_AWS_144:dev env + #checkov:skip=CKV_AWS_21:dev env +} + +resource "aws_s3_bucket" "osquery-status" { + bucket = "fleet-loadtest-osquery-status-archive" + acl = "private" + + lifecycle_rule { + enabled = true + expiration { + days = 1 + } + } + + server_side_encryption_configuration { + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "aws:kms" + } + } + } + #checkov:skip=CKV_AWS_18:dev env + #checkov:skip=CKV_AWS_144:dev env + #checkov:skip=CKV_AWS_21:dev env +} + +data "aws_iam_policy_document" "osquery_results_policy_doc" { + statement { + effect = "Allow" + actions = [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject" + ] + resources = [aws_s3_bucket.osquery-results.arn, "${aws_s3_bucket.osquery-results.arn}/*"] + } +} + +data "aws_iam_policy_document" "osquery_status_policy_doc" { + statement { + effect = "Allow" + actions = [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:PutObject" + ] + resources = [aws_s3_bucket.osquery-status.arn, "${aws_s3_bucket.osquery-status.arn}/*"] + } +} + +resource "aws_iam_policy" "firehose-results" { + name = "osquery_results_firehose_policy" + policy = data.aws_iam_policy_document.osquery_results_policy_doc.json +} + +resource "aws_iam_policy" "firehose-status" { + name = "osquery_status_firehose_policy" + policy = data.aws_iam_policy_document.osquery_status_policy_doc.json +} + +resource "aws_iam_role" "firehose-results" { + assume_role_policy = data.aws_iam_policy_document.osquery_firehose_assume_role.json +} + +resource "aws_iam_role" "firehose-status" { + assume_role_policy = data.aws_iam_policy_document.osquery_firehose_assume_role.json +} + +resource "aws_iam_role_policy_attachment" "firehose-results" { + policy_arn = aws_iam_policy.firehose-results.arn + role = aws_iam_role.firehose-results.name +} + +resource "aws_iam_role_policy_attachment" "firehose-status" { + policy_arn = aws_iam_policy.firehose-status.arn + role = aws_iam_role.firehose-status.name +} + +data "aws_iam_policy_document" "osquery_firehose_assume_role" { + statement { + effect = "Allow" + actions = ["sts:AssumeRole"] + principals { + identifiers = ["firehose.amazonaws.com"] + type = "Service" + } + } +} + +resource "aws_kinesis_firehose_delivery_stream" "osquery_results" { + name = "osquery_results" + destination = "s3" + + s3_configuration { + role_arn = aws_iam_role.firehose-results.arn + bucket_arn = aws_s3_bucket.osquery-results.arn + } +} + +resource "aws_kinesis_firehose_delivery_stream" "osquery_status" { + name = "osquery_status" + destination = "s3" + + s3_configuration { + role_arn = aws_iam_role.firehose-status.arn + bucket_arn = aws_s3_bucket.osquery-status.arn + } +} diff --git a/tools/loadtesting/terraform/locals.tf b/tools/loadtesting/terraform/locals.tf new file mode 100644 index 0000000000..b166d1c3f1 --- /dev/null +++ b/tools/loadtesting/terraform/locals.tf @@ -0,0 +1,17 @@ +locals { + name = "fleetdm" + prefix = "fleet" + domain_fleetdm = "loadtest.fleetdm.com" + domain_fleetctl = "loadtest.fleetctl.com" + additional_env_vars = [for k, v in merge({ + "FLEET_VULNERABILITIES_DATABASES_PATH" : "/home/fleet" + "FLEET_OSQUERY_ENABLE_ASYNC_HOST_PROCESSING" : "false" + "FLEET_LOGGING_DEBUG" : "true" + //"OTEL_EXPORTER_OTLP_ENDPOINT" : "http://10.10.2.193:8200" + //"FLEET_LOGGING_TRACING_ENABLED" : "true" + "ELASTIC_APM_SERVER_URL" : "http://10.10.2.193:8200" + "ELASTIC_APM_SERVICE_NAME" : "fleet" + "ELASTIC_APM_ENVIRONMENT" : "loadtest" + "ELASTIC_APM_TRANSACTION_SAMPLE_RATE" : "0.004" + }, var.fleet_config) : { name = k, value = v }] +} diff --git a/tools/loadtesting/terraform/main.tf b/tools/loadtesting/terraform/main.tf new file mode 100644 index 0000000000..b10d9e8b7b --- /dev/null +++ b/tools/loadtesting/terraform/main.tf @@ -0,0 +1,44 @@ +variable "region" { + default = "us-east-2" +} + +provider "aws" { + region = var.region + default_tags { + tags = { + environment = "loadtest" + terraform = "https://github.com/fleetdm/fleet/tree/main/tools/terraform" + state = "local" + } + } +} + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.74.0" + } + docker = { + source = "kreuzwerker/docker" + version = "~> 2.16.0" + } + } + backend "s3" { + bucket = "fleet-loadtesting-tfstate" + key = "loadtesting" + region = "us-east-2" + dynamodb_table = "fleet-loadtesting-tfstate" + } +} + +data "aws_caller_identity" "current" {} + +provider "docker" { + # Configuration options + registry_auth { + address = "${data.aws_caller_identity.current.account_id}.dkr.ecr.us-east-2.amazonaws.com" + username = data.aws_ecr_authorization_token.token.user_name + password = data.aws_ecr_authorization_token.token.password + } +} diff --git a/tools/loadtesting/terraform/outputs.tf b/tools/loadtesting/terraform/outputs.tf new file mode 100644 index 0000000000..ecec2a89f1 --- /dev/null +++ b/tools/loadtesting/terraform/outputs.tf @@ -0,0 +1,15 @@ +output "nameservers_fleetctl" { + value = aws_route53_zone.dogfood_fleetctl_com.name_servers +} + +output "nameservers_fleetdm" { + value = aws_route53_zone.dogfood_fleetdm_com.name_servers +} + +output "backend_security_group" { + value = aws_security_group.backend.arn +} + +output "private_subnets" { + value = module.vpc.private_subnet_arns +} \ No newline at end of file diff --git a/tools/loadtesting/terraform/r53.tf b/tools/loadtesting/terraform/r53.tf new file mode 100644 index 0000000000..9bf70b8ba5 --- /dev/null +++ b/tools/loadtesting/terraform/r53.tf @@ -0,0 +1,62 @@ +resource "aws_route53_zone" "dogfood_fleetctl_com" { + name = local.domain_fleetctl +} + +resource "aws_route53_zone" "dogfood_fleetdm_com" { + name = local.domain_fleetdm +} + +resource "aws_route53_record" "dogfood_fleetctl_com" { + zone_id = aws_route53_zone.dogfood_fleetctl_com.zone_id + name = local.domain_fleetctl + type = "A" + + alias { + name = aws_alb.main.dns_name + zone_id = aws_alb.main.zone_id + evaluate_target_health = false + } +} + +resource "aws_route53_record" "dogfood_fleetdm_com" { + zone_id = aws_route53_zone.dogfood_fleetdm_com.zone_id + name = local.domain_fleetdm + type = "A" + + alias { + name = aws_alb.main.dns_name + zone_id = aws_alb.main.zone_id + evaluate_target_health = false + } +} + +resource "aws_acm_certificate" "dogfood_fleetdm_com" { + domain_name = local.domain_fleetdm + validation_method = "DNS" + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_route53_record" "dogfood_fleetdm_com_validation" { + for_each = { + for dvo in aws_acm_certificate.dogfood_fleetdm_com.domain_validation_options : dvo.domain_name => { + name = dvo.resource_record_name + record = dvo.resource_record_value + type = dvo.resource_record_type + } + } + + allow_overwrite = true + name = each.value.name + records = [each.value.record] + ttl = 60 + type = each.value.type + zone_id = aws_route53_zone.dogfood_fleetdm_com.zone_id +} + +resource "aws_acm_certificate_validation" "dogfood_fleetdm_com" { + certificate_arn = aws_acm_certificate.dogfood_fleetdm_com.arn + validation_record_fqdns = [for record in aws_route53_record.dogfood_fleetdm_com_validation : record.fqdn] +} diff --git a/tools/loadtesting/terraform/rds.tf b/tools/loadtesting/terraform/rds.tf new file mode 100644 index 0000000000..ae04e1f45f --- /dev/null +++ b/tools/loadtesting/terraform/rds.tf @@ -0,0 +1,77 @@ +resource "random_password" "database_password" { + length = 16 + special = false +} + +resource "random_pet" "db_secret_postfix" { + length = 1 +} + +resource "aws_secretsmanager_secret" "database_password_secret" { + name = "/fleet/database/password/master-2-${random_pet.db_secret_postfix.id}" + kms_key_id = aws_kms_key.main.id +} + +resource "aws_secretsmanager_secret_version" "database_password_secret_version" { + secret_id = aws_secretsmanager_secret.database_password_secret.id + secret_string = random_password.database_password.result +} + +module "aurora_mysql" { + source = "terraform-aws-modules/rds-aurora/aws" + version = "5.3.0" + + name = "${local.name}-mysql-iam" + engine = "aurora-mysql" + engine_version = "5.7.mysql_aurora.2.10.0" + instance_type = "db.r5.2xlarge" + instance_type_replica = "db.r5.2xlarge" + + iam_database_authentication_enabled = true + storage_encrypted = true + username = "fleet" + password = random_password.database_password.result + create_random_password = false + database_name = "fleet" + enable_http_endpoint = false + performance_insights_enabled = true + enabled_cloudwatch_logs_exports = ["slowquery"] + + vpc_id = module.vpc.vpc_id + vpc_security_group_ids = [aws_security_group.backend.id] + subnets = module.vpc.database_subnets + create_security_group = true + allowed_cidr_blocks = module.vpc.private_subnets_cidr_blocks + + replica_count = 1 + replica_scale_enabled = true + replica_scale_min = 1 + replica_scale_max = 3 + snapshot_identifier = "arn:aws:rds:us-east-2:917007347864:cluster-snapshot:fleetdm-mysql-iam-final-prerebuild" + + monitoring_interval = 60 + iam_role_name = "${local.name}-rds-enhanced-monitoring" + iam_role_use_name_prefix = true + iam_role_description = "${local.name} RDS enhanced monitoring IAM role" + iam_role_path = "/autoscaling/" + iam_role_max_session_duration = 7200 + + apply_immediately = true + skip_final_snapshot = true + + db_parameter_group_name = aws_db_parameter_group.example_mysql.id + db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.example_mysql.id + +} + +resource "aws_db_parameter_group" "example_mysql" { + name = "${local.name}-aurora-db-mysql-parameter-group" + family = "aurora-mysql5.7" + description = "${local.name}-aurora-db-mysql-parameter-group" +} + +resource "aws_rds_cluster_parameter_group" "example_mysql" { + name = "${local.name}-aurora-mysql-cluster-parameter-group" + family = "aurora-mysql5.7" + description = "${local.name}-aurora-mysql-cluster-parameter-group" +} diff --git a/tools/loadtesting/terraform/readme.md b/tools/loadtesting/terraform/readme.md new file mode 100644 index 0000000000..bb7ddd3b43 --- /dev/null +++ b/tools/loadtesting/terraform/readme.md @@ -0,0 +1,12 @@ +## Terraform for Loadtesting Environment + +The interface into this code is designed to be minimal. +If you require changes beyond whats described here, contact @zwinnerman-fleetdm. + +### Deploying your code to the loadtesting environment +1. Initialize your terraform environment with `terraform init` +2. Apply terraform with your branch name with `terraform apply -var tag=BRANCH_NAME` + +### Running migrations +After applying terraform with the commands above: +`aws ecs run-task --region us-east-2 --cluster fleet-backend --task-definition fleet-migrate:"$(terraform output -raw fleet_migration_revision)" --launch-type FARGATE --network-configuration "awsvpcConfiguration={subnets="$(terraform output -raw fleet_migration_subnets)",securityGroups="$(terraform output -raw fleet_migration_security_groups)"}"` diff --git a/tools/loadtesting/terraform/redis.tf b/tools/loadtesting/terraform/redis.tf new file mode 100644 index 0000000000..8988d2cb96 --- /dev/null +++ b/tools/loadtesting/terraform/redis.tf @@ -0,0 +1,59 @@ +variable "maintenance_window" { + default = "" +} +variable "engine_version" { + default = "5.0.6" +} +variable "number_cache_clusters" { + default = 3 +} +variable "redis_instance" { + default = "cache.m5.large" +} +resource "aws_elasticache_replication_group" "default" { + availability_zones = ["us-east-2a", "us-east-2b", "us-east-2c"] + engine = "redis" + parameter_group_name = "default.redis5.0" + subnet_group_name = module.vpc.elasticache_subnet_group_name + security_group_ids = [aws_security_group.redis.id, aws_security_group.backend.id] + replication_group_id = "fleetdm-redis" + number_cache_clusters = var.number_cache_clusters + node_type = var.redis_instance + engine_version = var.engine_version + port = "6379" + maintenance_window = var.maintenance_window + snapshot_retention_limit = 0 + automatic_failover_enabled = true + at_rest_encryption_enabled = false + transit_encryption_enabled = false + apply_immediately = true + replication_group_description = "fleetdm-redis" + +} + +resource "aws_security_group" "redis" { + name = local.security_group_name + vpc_id = module.vpc.vpc_id +} + +locals { + security_group_name = "${local.prefix}-elasticache-redis" +} + +resource "aws_security_group_rule" "ingress" { + type = "ingress" + from_port = "6379" + to_port = "6379" + protocol = "tcp" + cidr_blocks = module.vpc.private_subnets_cidr_blocks + security_group_id = aws_security_group.redis.id +} + +resource "aws_security_group_rule" "egress" { + type = "egress" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_security_group.redis.id +} diff --git a/tools/loadtesting/terraform/state/state.yml b/tools/loadtesting/terraform/state/state.yml new file mode 100644 index 0000000000..956bdc1207 --- /dev/null +++ b/tools/loadtesting/terraform/state/state.yml @@ -0,0 +1,13 @@ +--- +- connection: local + gather_facts: false + hosts: 127.0.0.1 + tasks: + - amazon.aws.s3_bucket: + name: fleet-loadtesting-tfstate + state: present + - community.aws.dynamodb_table: + name: fleet-loadtesting-tfstate + hash_key_name: LockID + hash_key_type: STRING + billing_mode: PAY_PER_REQUEST diff --git a/tools/loadtesting/terraform/variables.tf b/tools/loadtesting/terraform/variables.tf new file mode 100644 index 0000000000..28b592adfc --- /dev/null +++ b/tools/loadtesting/terraform/variables.tf @@ -0,0 +1,9 @@ +variable "tag" { + description = "The tag to deploy. This would be the same as the branch name" +} + +variable "fleet_config" { + description = "The configuration to use for fleet itself, gets translated as environment variables" + type = map(string) + default = {} +} diff --git a/tools/loadtesting/terraform/vpc.tf b/tools/loadtesting/terraform/vpc.tf new file mode 100644 index 0000000000..a5346e57eb --- /dev/null +++ b/tools/loadtesting/terraform/vpc.tf @@ -0,0 +1,25 @@ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "3.12.0" + + name = "fleet-vpc" + cidr = "10.10.0.0/16" + + azs = ["us-east-2a", "us-east-2b", "us-east-2c"] + private_subnets = ["10.10.1.0/24", "10.10.2.0/24", "10.10.3.0/24"] + public_subnets = ["10.10.11.0/24", "10.10.12.0/24", "10.10.13.0/24"] + database_subnets = ["10.10.21.0/24", "10.10.22.0/24", "10.10.23.0/24"] + elasticache_subnets = ["10.10.31.0/24", "10.10.32.0/24", "10.10.33.0/24"] + + create_database_subnet_group = true + create_database_subnet_route_table = true + + create_elasticache_subnet_group = true + create_elasticache_subnet_route_table = true + + enable_vpn_gateway = false + one_nat_gateway_per_az = false + + single_nat_gateway = true + enable_nat_gateway = true +}