Add support for reading private_key from AWS Secrets Manager (#31134)

Adds support for reading server `private_key` from AWS Secrets Manager.
Combined with #31075, this should allow removing all common sensitive
secrets from the environment/config (if I missed any let me know). This
works with localstack for local development (set
`AWS_ENDPOINT_URL=$LOCALSTACK_URL`, `AWS_ACCESS_KEY_ID=test`, and
`AWS_SECRET_ACCESS_KEY=test`).

I did not include config options for `AWS_ACCESS_KEY_ID` and
`AWS_SECRET_ACCESS_KEY` because they are a bad practice vs role
credentials and defeat the purpose of this feature which is to remove
secrets from the environment/config.

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

<!-- Note that API documentation changes are now addressed by the
product design team. -->

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.
- [x] Added/updated automated tests
- [x] Manual QA for all new/changed functionality

---------

Co-authored-by: Scott Gress <scott@fleetdm.com>

server/config/config.go

@@ -95,20 +95,23 @@ const (
 
 // ServerConfig defines configs related to the Fleet server
 type ServerConfig struct {
-	Address                     string
-	Cert                        string
-	Key                         string
-	TLS                         bool
-	TLSProfile                  string        `yaml:"tls_compatibility"`
-	URLPrefix                   string        `yaml:"url_prefix"`
-	Keepalive                   bool          `yaml:"keepalive"`
-	SandboxEnabled              bool          `yaml:"sandbox_enabled"`
-	WebsocketsAllowUnsafeOrigin bool          `yaml:"websockets_allow_unsafe_origin"`
-	FrequentCleanupsEnabled     bool          `yaml:"frequent_cleanups_enabled"`
-	ForceH2C                    bool          `yaml:"force_h2c"`
-	PrivateKey                  string        `yaml:"private_key"`
-	VPPVerifyTimeout            time.Duration `yaml:"vpp_verify_timeout"`
-	VPPVerifyRequestDelay       time.Duration `yaml:"vpp_verify_request_delay"`
+	Address                          string
+	Cert                             string
+	Key                              string
+	TLS                              bool
+	TLSProfile                       string        `yaml:"tls_compatibility"`
+	URLPrefix                        string        `yaml:"url_prefix"`
+	Keepalive                        bool          `yaml:"keepalive"`
+	SandboxEnabled                   bool          `yaml:"sandbox_enabled"`
+	WebsocketsAllowUnsafeOrigin      bool          `yaml:"websockets_allow_unsafe_origin"`
+	FrequentCleanupsEnabled          bool          `yaml:"frequent_cleanups_enabled"`
+	ForceH2C                         bool          `yaml:"force_h2c"`
+	PrivateKey                       string        `yaml:"private_key"`
+	PrivateKeySecretArn              string        `yaml:"private_key_secret_arn"`
+	PrivateKeySecretSTSAssumeRoleArn string        `yaml:"private_key_secret_sts_assume_role_arn"`
+	PrivateKeySecretSTSExternalID    string        `yaml:"private_key_secret_sts_external_id"`
+	VPPVerifyTimeout                 time.Duration `yaml:"vpp_verify_timeout"`
+	VPPVerifyRequestDelay            time.Duration `yaml:"vpp_verify_request_delay"`
 }
 
 func (s *ServerConfig) DefaultHTTPServer(ctx context.Context, handler http.Handler) *http.Server {
@@ -1121,6 +1124,9 @@ func (man Manager) addConfigs() {
 	man.addConfigBool("server.frequent_cleanups_enabled", false, "Enable frequent cleanups of expired data (15 minute interval)")
 	man.addConfigBool("server.force_h2c", false, "Force the fleet server to use HTTP2 cleartext aka h2c (ignored if using TLS)")
 	man.addConfigString("server.private_key", "", "Used for encrypting sensitive data, such as MDM certificates.")
+	man.addConfigString("server.private_key_secret_arn", "", "ARN of AWS Secrets Manager secret containing server private key")
+	man.addConfigString("server.private_key_secret_sts_assume_role_arn", "", "ARN of role to assume for accessing private key secret")
+	man.addConfigString("server.private_key_secret_sts_external_id", "", "External ID for STS role assumption when accessing private key secret")
 	man.addConfigDuration("server.vpp_verify_timeout", 10*time.Minute, "Maximum amout of time to wait for VPP app install verification")
 	man.addConfigDuration("server.vpp_verify_request_delay", 5*time.Second, "Delay in between requests to verify VPP app installs")
 
@@ -1548,20 +1554,23 @@ func (man Manager) LoadConfig() FleetConfig {
 			StsExternalID:             man.getConfigString("redis.sts_external_id"),
 		},
 		Server: ServerConfig{
-			Address:                     man.getConfigString("server.address"),
-			Cert:                        man.getConfigString("server.cert"),
-			Key:                         man.getConfigString("server.key"),
-			TLS:                         man.getConfigBool("server.tls"),
-			TLSProfile:                  man.getConfigTLSProfile(),
-			URLPrefix:                   man.getConfigString("server.url_prefix"),
-			Keepalive:                   man.getConfigBool("server.keepalive"),
-			SandboxEnabled:              man.getConfigBool("server.sandbox_enabled"),
-			WebsocketsAllowUnsafeOrigin: man.getConfigBool("server.websockets_allow_unsafe_origin"),
-			FrequentCleanupsEnabled:     man.getConfigBool("server.frequent_cleanups_enabled"),
-			ForceH2C:                    man.getConfigBool("server.force_h2c"),
-			PrivateKey:                  man.getConfigString("server.private_key"),
-			VPPVerifyTimeout:            man.getConfigDuration("server.vpp_verify_timeout"),
-			VPPVerifyRequestDelay:       man.getConfigDuration("server.vpp_verify_request_delay"),
+			Address:                          man.getConfigString("server.address"),
+			Cert:                             man.getConfigString("server.cert"),
+			Key:                              man.getConfigString("server.key"),
+			TLS:                              man.getConfigBool("server.tls"),
+			TLSProfile:                       man.getConfigTLSProfile(),
+			URLPrefix:                        man.getConfigString("server.url_prefix"),
+			Keepalive:                        man.getConfigBool("server.keepalive"),
+			SandboxEnabled:                   man.getConfigBool("server.sandbox_enabled"),
+			WebsocketsAllowUnsafeOrigin:      man.getConfigBool("server.websockets_allow_unsafe_origin"),
+			FrequentCleanupsEnabled:          man.getConfigBool("server.frequent_cleanups_enabled"),
+			ForceH2C:                         man.getConfigBool("server.force_h2c"),
+			PrivateKey:                       man.getConfigString("server.private_key"),
+			PrivateKeySecretArn:              man.getConfigString("server.private_key_secret_arn"),
+			PrivateKeySecretSTSAssumeRoleArn: man.getConfigString("server.private_key_secret_sts_assume_role_arn"),
+			PrivateKeySecretSTSExternalID:    man.getConfigString("server.private_key_secret_sts_external_id"),
+			VPPVerifyTimeout:                 man.getConfigDuration("server.vpp_verify_timeout"),
+			VPPVerifyRequestDelay:            man.getConfigDuration("server.vpp_verify_request_delay"),
 		},
 		Auth: AuthConfig{
 			BcryptCost:                  man.getConfigInt("auth.bcrypt_cost"),

server/config/secrets_manager.go

@@ -0,0 +1,130 @@
+package config
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand"
+	"strings"
+	"time"
+
+	aws_config "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
+	"github.com/aws/aws-sdk-go-v2/service/secretsmanager/types"
+	"github.com/fleetdm/fleet/v4/server/aws_common"
+)
+
+// SecretsManagerClient interface for dependency injection and testing
+type SecretsManagerClient interface {
+	GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput,
+		optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error)
+}
+
+// parseRegionFromSecretARN extracts the AWS region from a Secrets Manager ARN
+func parseRegionFromSecretARN(arn string) (string, error) {
+	// ARN format: arn:aws:secretsmanager:region:account:secret:name
+	parts := strings.Split(arn, ":")
+	if len(parts) < 6 || parts[0] != "arn" || parts[1] != "aws" || parts[2] != "secretsmanager" {
+		return "", fmt.Errorf("invalid Secrets Manager ARN format: %s", arn)
+	}
+
+	region := parts[3]
+	if region == "" {
+		return "", fmt.Errorf("region not found in ARN: %s", arn)
+	}
+
+	return region, nil
+}
+
+// retrieveSecretWithRetry retrieves the secret from AWS with retry logic
+func retrieveSecretWithRetry(ctx context.Context, client SecretsManagerClient, secretArn string) (string, error) {
+	const maxRetries = 3
+	var lastErr error
+
+	for attempt := 0; attempt < maxRetries; attempt++ {
+		if attempt > 0 {
+			// Exponential backoff with jitter: base 100ms with ±50% randomization
+			baseBackoff := time.Duration(100*(1<<uint(attempt-1))) * time.Millisecond    // #nosec G115 - attempt is bounded by maxRetries
+			jitter := time.Duration(rand.Float64()*float64(baseBackoff)) - baseBackoff/2 // #nosec G404 - not security sensitive
+			backoff := baseBackoff + jitter
+			select {
+			case <-ctx.Done():
+				return "", ctx.Err()
+			case <-time.After(backoff):
+			}
+		}
+
+		input := &secretsmanager.GetSecretValueInput{
+			SecretId: &secretArn,
+		}
+
+		output, err := client.GetSecretValue(ctx, input)
+		if err != nil {
+			lastErr = err
+
+			// Don't retry certain errors
+			var notFoundErr *types.ResourceNotFoundException
+			var unauthorizedErr *types.InvalidRequestException
+			var invalidParamErr *types.InvalidParameterException
+
+			if errors.As(err, &notFoundErr) {
+				return "", fmt.Errorf("secret not found: %s", secretArn)
+			}
+			if errors.As(err, &unauthorizedErr) {
+				return "", fmt.Errorf("access denied to secret: %s", secretArn)
+			}
+			if errors.As(err, &invalidParamErr) {
+				return "", fmt.Errorf("invalid secret ARN: %s", secretArn)
+			}
+
+			// Retry for other errors (network issues, throttling, etc.)
+			continue
+		}
+
+		// Extract secret value
+		if output.SecretString != nil {
+			return *output.SecretString, nil
+		}
+
+		if output.SecretBinary != nil {
+			return "", fmt.Errorf("secret %s contains binary data, expected string", secretArn)
+		}
+
+		return "", fmt.Errorf("secret %s contains no data", secretArn)
+	}
+
+	return "", fmt.Errorf("failed to retrieve secret after %d attempts: %w", maxRetries, lastErr)
+}
+
+// RetrieveSecretsManagerSecret retrieves a secret from AWS Secrets Manager
+// with support for STS assume role authentication
+func RetrieveSecretsManagerSecret(ctx context.Context, secretArn, assumeRoleArn, externalID string) (string, error) {
+	return RetrieveSecretsManagerSecretWithOptions(ctx, secretArn, assumeRoleArn, externalID)
+}
+
+// RetrieveSecretsManagerSecretWithOptions retrieves a secret from AWS Secrets Manager
+// with custom AWS config options (useful for testing with LocalStack)
+func RetrieveSecretsManagerSecretWithOptions(ctx context.Context, secretArn, assumeRoleArn, externalID string, opts ...func(*aws_config.LoadOptions) error) (string, error) {
+	region, err := parseRegionFromSecretARN(secretArn)
+	if err != nil {
+		return "", fmt.Errorf("invalid secret ARN: %w", err)
+	}
+
+	configOpts := []func(*aws_config.LoadOptions) error{aws_config.WithRegion(region)}
+	configOpts = append(configOpts, opts...)
+	cfg, err := aws_config.LoadDefaultConfig(ctx, configOpts...)
+	if err != nil {
+		return "", fmt.Errorf("failed to load AWS config: %w", err)
+	}
+
+	if assumeRoleArn != "" {
+		cfg, err = aws_common.ConfigureAssumeRoleProvider(cfg, nil, assumeRoleArn, externalID)
+		if err != nil {
+			return "", fmt.Errorf("failed to configure assume role: %w", err)
+		}
+	}
+
+	client := secretsmanager.NewFromConfig(cfg)
+
+	return retrieveSecretWithRetry(ctx, client, secretArn)
+}

server/config/secrets_manager_test.go

@@ -0,0 +1,271 @@
+package config
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	aws_config "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
+	"github.com/aws/aws-sdk-go-v2/service/secretsmanager/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+// mockSecretsManagerClient is a mock implementation of the SecretsManagerClient interface
+type mockSecretsManagerClient struct {
+	mock.Mock
+}
+
+// GetSecretValue mocks the AWS Secrets Manager GetSecretValue operation
+func (m *mockSecretsManagerClient) GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) {
+	args := m.Called(ctx, params)
+	if args.Get(0) == nil {
+		return nil, args.Error(1)
+	}
+	return args.Get(0).(*secretsmanager.GetSecretValueOutput), args.Error(1)
+}
+
+func TestParseRegionFromARN(t *testing.T) {
+	testCases := []struct {
+		name        string
+		arn         string
+		expectedReg string
+		expectError bool
+	}{
+		{
+			name:        "valid ARN",
+			arn:         "arn:aws:secretsmanager:us-west-2:123456789012:secret:fleet-private-key-abc123",
+			expectedReg: "us-west-2",
+			expectError: false,
+		},
+		{
+			name:        "valid ARN different region",
+			arn:         "arn:aws:secretsmanager:eu-central-1:123456789012:secret:my-secret-def456",
+			expectedReg: "eu-central-1",
+			expectError: false,
+		},
+		{
+			name:        "invalid ARN format",
+			arn:         "invalid-arn-format",
+			expectedReg: "",
+			expectError: true,
+		},
+		{
+			name:        "wrong service",
+			arn:         "arn:aws:s3:us-west-2:123456789012:bucket/my-bucket",
+			expectedReg: "",
+			expectError: true,
+		},
+		{
+			name:        "empty region",
+			arn:         "arn:aws:secretsmanager::123456789012:secret:my-secret",
+			expectedReg: "",
+			expectError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			region, err := parseRegionFromSecretARN(tc.arn)
+			if tc.expectError {
+				assert.Error(t, err)
+				assert.Empty(t, region)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tc.expectedReg, region)
+			}
+		})
+	}
+}
+
+func TestRetrieveSecretWithRetry_Success(t *testing.T) {
+	mockClient := &mockSecretsManagerClient{}
+	expectedKey := "test-32-byte-key-for-aes-encryption"
+	secretArn := "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-secret" // #nosec G101 - test data
+
+	mockClient.On("GetSecretValue", mock.Anything, mock.MatchedBy(func(input *secretsmanager.GetSecretValueInput) bool {
+		return input.SecretId != nil && *input.SecretId == secretArn
+	})).Return(&secretsmanager.GetSecretValueOutput{
+		SecretString: &expectedKey,
+	}, nil)
+
+	key, err := retrieveSecretWithRetry(context.Background(), mockClient, secretArn)
+
+	require.NoError(t, err)
+	assert.Equal(t, expectedKey, key)
+	mockClient.AssertExpectations(t)
+}
+
+func TestRetrieveSecretWithRetry_BinarySecret(t *testing.T) {
+	mockClient := &mockSecretsManagerClient{}
+	secretArn := "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-secret" // #nosec G101 - test data
+	binaryData := []byte("binary-data")
+
+	mockClient.On("GetSecretValue", mock.Anything, mock.MatchedBy(func(input *secretsmanager.GetSecretValueInput) bool {
+		return input.SecretId != nil && *input.SecretId == secretArn
+	})).Return(&secretsmanager.GetSecretValueOutput{
+		SecretBinary: binaryData,
+	}, nil)
+
+	_, err := retrieveSecretWithRetry(context.Background(), mockClient, secretArn)
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "contains binary data, expected string")
+	mockClient.AssertExpectations(t)
+}
+
+func TestRetrieveSecretWithRetry_EmptySecret(t *testing.T) {
+	mockClient := &mockSecretsManagerClient{}
+	secretArn := "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-secret" // #nosec G101 - test data
+
+	mockClient.On("GetSecretValue", mock.Anything, mock.MatchedBy(func(input *secretsmanager.GetSecretValueInput) bool {
+		return input.SecretId != nil && *input.SecretId == secretArn
+	})).Return(&secretsmanager.GetSecretValueOutput{
+		// Both SecretString and SecretBinary are nil
+	}, nil)
+
+	_, err := retrieveSecretWithRetry(context.Background(), mockClient, secretArn)
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "contains no data")
+	mockClient.AssertExpectations(t)
+}
+
+func TestRetrieveSecretWithRetry_ErrorHandling(t *testing.T) {
+	testCases := []struct {
+		name        string
+		mockError   error
+		expectedErr string
+		shouldRetry bool
+	}{
+		{
+			name:        "ResourceNotFound",
+			mockError:   &types.ResourceNotFoundException{},
+			expectedErr: "secret not found",
+			shouldRetry: false,
+		},
+		{
+			name:        "InvalidRequest",
+			mockError:   &types.InvalidRequestException{},
+			expectedErr: "access denied",
+			shouldRetry: false,
+		},
+		{
+			name:        "InvalidParameter",
+			mockError:   &types.InvalidParameterException{},
+			expectedErr: "invalid secret ARN",
+			shouldRetry: false,
+		},
+		{
+			name:        "NetworkError",
+			mockError:   errors.New("network timeout"),
+			expectedErr: "failed to retrieve secret after 3 attempts",
+			shouldRetry: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			mockClient := &mockSecretsManagerClient{}
+			secretArn := "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-secret" // #nosec G101 - test data, not real credentials
+
+			if tc.shouldRetry {
+				// Should be called 3 times for retryable errors
+				mockClient.On("GetSecretValue", mock.Anything, mock.Anything).Return(
+					(*secretsmanager.GetSecretValueOutput)(nil), tc.mockError).Times(3)
+			} else {
+				// Should only be called once for non-retryable errors
+				mockClient.On("GetSecretValue", mock.Anything, mock.Anything).Return(
+					(*secretsmanager.GetSecretValueOutput)(nil), tc.mockError).Once()
+			}
+
+			_, err := retrieveSecretWithRetry(context.Background(), mockClient, secretArn)
+
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.expectedErr)
+			mockClient.AssertExpectations(t)
+		})
+	}
+}
+
+func TestRetrieveSecretWithRetry_ContextCancellation(t *testing.T) {
+	mockClient := &mockSecretsManagerClient{}
+	secretArn := "arn:aws:secretsmanager:us-west-2:123456789012:secret:test-secret" // #nosec G101 - test data, not real credentials
+
+	// Create a context that's already cancelled
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	// Mock the first call to return a retryable error
+	mockClient.On("GetSecretValue", mock.Anything, mock.Anything).Return(
+		(*secretsmanager.GetSecretValueOutput)(nil), errors.New("network error")).Once()
+
+	_, err := retrieveSecretWithRetry(ctx, mockClient, secretArn)
+
+	require.Error(t, err)
+	assert.Equal(t, context.Canceled, err)
+	mockClient.AssertExpectations(t)
+}
+
+func TestRetrieveSecretsManagerSecret_LocalStack(t *testing.T) {
+	if os.Getenv("LOCALSTACK_URL") == "" {
+		t.Skip("LOCALSTACK_URL not set, skipping LocalStack integration test")
+	}
+
+	ctx := context.Background()
+
+	localStackURL := os.Getenv("LOCALSTACK_URL")
+	if localStackURL == "" {
+		localStackURL = "http://localhost:4566"
+	}
+
+	localStackOpts := []func(*aws_config.LoadOptions) error{
+		aws_config.WithBaseEndpoint(localStackURL),
+		aws_config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", "")),
+	}
+
+	// Configure LocalStack client
+	cfg, err := aws_config.LoadDefaultConfig(ctx, append(localStackOpts, aws_config.WithRegion("us-east-1"))...)
+	require.NoError(t, err)
+	client := secretsmanager.NewFromConfig(cfg)
+
+	secretName := "fleet-test-private-key-localstack"
+	privateKey := "test-key-exactly-32-bytes-long!"
+	secretArn := fmt.Sprintf("arn:aws:secretsmanager:us-east-1:000000000000:secret:%s", secretName)
+
+	// Clean up any existing secret
+	_, _ = client.DeleteSecret(ctx, &secretsmanager.DeleteSecretInput{
+		SecretId:                   &secretName,
+		ForceDeleteWithoutRecovery: aws.Bool(true),
+	})
+
+	_, err = client.CreateSecret(ctx, &secretsmanager.CreateSecretInput{
+		Name:         &secretName,
+		SecretString: &privateKey,
+		Description:  aws.String("password"),
+	})
+	require.NoError(t, err, "Failed to create secret in LocalStack")
+
+	// Clean up after test
+	defer func() {
+		_, _ = client.DeleteSecret(ctx, &secretsmanager.DeleteSecretInput{
+			SecretId:                   &secretName,
+			ForceDeleteWithoutRecovery: aws.Bool(true),
+		})
+	}()
+	retrievedKey, err := RetrieveSecretsManagerSecretWithOptions(ctx, secretArn, "", "", localStackOpts...)
+	require.NoError(t, err)
+	assert.Equal(t, privateKey, retrievedKey)
+
+	// Test with invalid ARN
+	invalidArn := "arn:aws:secretsmanager:us-east-1:000000000000:secret:nonexistent-secret"
+	_, err = RetrieveSecretsManagerSecretWithOptions(ctx, invalidArn, "", "", localStackOpts...)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "secret not found")
+}