Added migration and secret variables API. (#24594)

#24545 

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

<!-- Note that API documentation changes are now addressed by the
product design team. -->

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/Committing-Changes.md#changes-files)
for more information.
- [x] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements)
- [x] Added/updated tests
- [x] If database migrations are included, checked table schema to
confirm autoupdate
- For database migrations:
- [x] Ensured the correct collation is explicitly set for character
columns (`COLLATE utf8mb4_unicode_ci`).
- [x] Manual QA for all new/changed functionality

changes/23238-use-secrets-in-scripts-profiles

@@ -0,0 +1,2 @@
+Added ability to use secrets ($FLEET_SECRET_NAME) in scripts and profiles.
+- Added `/fleet/spec/secret_variables` API endpoint.

server/authz/policy.rego

@@ -1008,3 +1008,14 @@ allow {
   team_role(subject, object.team_id) == [admin, maintainer, observer_plus, observer][_]
   action == read
 }
+
+##
+# Secret variables
+##
+
+# Global admins, maintainers, and gitops can write secret variables.
+allow {
+  object.type == "secret_variable"
+  subject.global_role == [admin, maintainer, gitops][_]
+  action == write
+}

server/datastore/mysql/migrations/tables/20241209164540_AddSecretVariablesTable.go

@@ -0,0 +1,33 @@
+package tables
+
+import (
+	"database/sql"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20241209164540, Down_20241209164540)
+}
+
+func Up_20241209164540(tx *sql.Tx) error {
+
+	if tableExists(tx, "secret_variables") {
+		return nil
+	}
+
+	_, err := tx.Exec(`
+		CREATE TABLE secret_variables (
+		id INT UNSIGNED NOT NULL AUTO_INCREMENT,
+		name VARCHAR(255) NOT NULL,
+		value BLOB NOT NULL, -- 64KB max value size
+		created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+		updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+		PRIMARY KEY (id),
+		CONSTRAINT idx_secret_variables_name UNIQUE (name)
+	) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci`,
+	)
+	return err
+}
+
+func Down_20241209164540(_ *sql.Tx) error {
+	return nil
+}

server/datastore/mysql/secret_variables.go

@@ -0,0 +1,70 @@
+package mysql
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/jmoiron/sqlx"
+)
+
+func (ds *Datastore) UpsertSecretVariables(ctx context.Context, secretVariables []fleet.SecretVariable) error {
+	if len(secretVariables) == 0 {
+		return nil
+	}
+
+	values := strings.TrimSuffix(strings.Repeat("(?,?),", len(secretVariables)), ",")
+
+	stmt := fmt.Sprintf(`
+		INSERT INTO secret_variables (name, value)
+		VALUES %s
+		ON DUPLICATE KEY UPDATE value = VALUES(value)`, values)
+
+	args := make([]interface{}, 0, len(secretVariables)*2)
+	for _, secretVariable := range secretVariables {
+		valueEncrypted, err := encrypt([]byte(secretVariable.Value), ds.serverPrivateKey)
+		if err != nil {
+			return ctxerr.Wrap(ctx, err, "encrypt secret value with server private key")
+		}
+		args = append(args, secretVariable.Name, valueEncrypted)
+	}
+
+	if _, err := ds.writer(ctx).ExecContext(ctx, stmt, args...); err != nil {
+		return ctxerr.Wrap(ctx, err, "upsert secret variables")
+	}
+
+	return nil
+}
+
+func (ds *Datastore) GetSecretVariables(ctx context.Context, names []string) ([]fleet.SecretVariable, error) {
+	if len(names) == 0 {
+		return nil, nil
+	}
+
+	stmt, args, err := sqlx.In(`
+		SELECT name, value
+		FROM secret_variables
+		WHERE name IN (?)`, names)
+	if err != nil {
+		return nil, ctxerr.Wrap(ctx, err, "build secret variables query")
+	}
+
+	var secretVariables []fleet.SecretVariable
+
+	err = sqlx.SelectContext(ctx, ds.reader(ctx), &secretVariables, stmt, args...)
+	if err != nil {
+		return nil, ctxerr.Wrap(ctx, err, "get secret variables")
+	}
+
+	for i, secretVariable := range secretVariables {
+		valueDecrypted, err := decrypt([]byte(secretVariable.Value), ds.serverPrivateKey)
+		if err != nil {
+			return nil, ctxerr.Wrap(ctx, err, "decrypt secret value with server private key")
+		}
+		secretVariables[i].Value = string(valueDecrypted)
+	}
+
+	return secretVariables, nil
+}

server/datastore/mysql/secret_variables_test.go

@@ -0,0 +1,72 @@
+package mysql
+
+import (
+	"context"
+	"testing"
+
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSecretVariables(t *testing.T) {
+	ds := CreateMySQLDS(t)
+
+	cases := []struct {
+		name string
+		fn   func(t *testing.T, ds *Datastore)
+	}{
+		{"UpsertSecretVariables", testUpsertSecretVariables},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			defer TruncateTables(t, ds)
+			c.fn(t, ds)
+		})
+	}
+}
+
+func testUpsertSecretVariables(t *testing.T, ds *Datastore) {
+	ctx := context.Background()
+	err := ds.UpsertSecretVariables(ctx, nil)
+	assert.NoError(t, err)
+	results, err := ds.GetSecretVariables(ctx, nil)
+	assert.NoError(t, err)
+	assert.Empty(t, results)
+
+	secretMap := map[string]string{
+		"test1": "testValue1",
+		"test2": "testValue2",
+		"test3": "testValue3",
+	}
+	createExpectedSecrets := func() []fleet.SecretVariable {
+		secrets := make([]fleet.SecretVariable, 0, len(secretMap))
+		for name, value := range secretMap {
+			secrets = append(secrets, fleet.SecretVariable{Name: name, Value: value})
+		}
+		return secrets
+	}
+	secrets := createExpectedSecrets()
+	err = ds.UpsertSecretVariables(ctx, secrets)
+	assert.NoError(t, err)
+
+	results, err = ds.GetSecretVariables(ctx, []string{"test1", "test2", "test3"})
+	assert.NoError(t, err)
+	assert.Len(t, results, 3)
+	for _, result := range results {
+		assert.Equal(t, secretMap[result.Name], result.Value)
+	}
+
+	// Update a secret
+	secretMap["test2"] = "newTestValue2"
+	err = ds.UpsertSecretVariables(ctx, []fleet.SecretVariable{
+		{Name: "test2", Value: secretMap["test2"]},
+	})
+	assert.NoError(t, err)
+	results, err = ds.GetSecretVariables(ctx, []string{"test2"})
+	assert.NoError(t, err)
+	require.Len(t, results, 1)
+	assert.Equal(t, "test2", results[0].Name)
+	assert.Equal(t, secretMap[results[0].Name], results[0].Value)
+
+}

server/fleet/datastore.go

@@ -1876,6 +1876,15 @@ type Datastore interface {
 
 	// GetSoftwareTitleIDByMaintainedAppID returns the software title ID for the given app ID.
 	GetSoftwareTitleIDByMaintainedAppID(ctx context.Context, appID uint, teamID *uint) (uint, error)
+
+	// /////////////////////////////////////////////////////////////////////////////
+	// Secret variables
+
+	// UpsertSecretVariables inserts or updates secret variables in the database.
+	UpsertSecretVariables(ctx context.Context, secretVariables []SecretVariable) error
+
+	// GetSecretVariables retrieves secret variables from the database.
+	GetSecretVariables(ctx context.Context, names []string) ([]SecretVariable, error)
 }
 
 // MDMAppleStore wraps nanomdm's storage and adds methods to deal with

server/fleet/secret_variables.go

@@ -0,0 +1,10 @@
+package fleet
+
+type SecretVariable struct {
+	Name  string `json:"name" db:"name"`
+	Value string `json:"value" db:"value"`
+}
+
+func (h SecretVariable) AuthzType() string {
+	return "secret_variable"
+}

server/fleet/service.go

@@ -1174,6 +1174,12 @@ type Service interface {
 
 	// CalendarWebhook handles incoming calendar callback requests.
 	CalendarWebhook(ctx context.Context, eventUUID string, channelID string, resourceState string) error
+
+	// /////////////////////////////////////////////////////////////////////////////
+	// Secret variables
+
+	// CreateSecretVariables creates secret variables for scripts and profiles.
+	CreateSecretVariables(ctx context.Context, secretVariables []SecretVariable) error
 }
 
 type KeyValueStore interface {

server/mock/datastore_mock.go

@@ -1175,6 +1175,10 @@ type CleanUpMDMManagedCertificatesFunc func(ctx context.Context) error
 
 type GetSoftwareTitleIDByMaintainedAppIDFunc func(ctx context.Context, appID uint, teamID *uint) (uint, error)
 
+type UpsertSecretVariablesFunc func(ctx context.Context, secretVariables []fleet.SecretVariable) error
+
+type GetSecretVariablesFunc func(ctx context.Context, names []string) ([]fleet.SecretVariable, error)
+
 type DataStore struct {
 	HealthCheckFunc        HealthCheckFunc
 	HealthCheckFuncInvoked bool
@@ -2907,6 +2911,12 @@ type DataStore struct {
 	GetSoftwareTitleIDByMaintainedAppIDFunc        GetSoftwareTitleIDByMaintainedAppIDFunc
 	GetSoftwareTitleIDByMaintainedAppIDFuncInvoked bool
 
+	UpsertSecretVariablesFunc        UpsertSecretVariablesFunc
+	UpsertSecretVariablesFuncInvoked bool
+
+	GetSecretVariablesFunc        GetSecretVariablesFunc
+	GetSecretVariablesFuncInvoked bool
+
 	mu sync.Mutex
 }
 
@@ -6948,3 +6958,17 @@ func (s *DataStore) GetSoftwareTitleIDByMaintainedAppID(ctx context.Context, app
 	s.mu.Unlock()
 	return s.GetSoftwareTitleIDByMaintainedAppIDFunc(ctx, appID, teamID)
 }
+
+func (s *DataStore) UpsertSecretVariables(ctx context.Context, secretVariables []fleet.SecretVariable) error {
+	s.mu.Lock()
+	s.UpsertSecretVariablesFuncInvoked = true
+	s.mu.Unlock()
+	return s.UpsertSecretVariablesFunc(ctx, secretVariables)
+}
+
+func (s *DataStore) GetSecretVariables(ctx context.Context, names []string) ([]fleet.SecretVariable, error) {
+	s.mu.Lock()
+	s.GetSecretVariablesFuncInvoked = true
+	s.mu.Unlock()
+	return s.GetSecretVariablesFunc(ctx, names)
+}

server/service/handler.go

@@ -525,6 +525,9 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC
 	// Generative AI
 	ue.POST("/api/_version_/fleet/autofill/policy", autofillPoliciesEndpoint, autofillPoliciesRequest{})
 
+	// Secret variables
+	ue.PUT("/api/_version_/fleet/spec/secret_variables", secretVariablesEndpoint, secretVariablesRequest{})
+
 	// Only Fleet MDM specific endpoints should be within the root /mdm/ path.
 	// NOTE: remember to update
 	// `service.mdmConfigurationRequiredEndpoints` when you add an

server/service/integration_core_test.go

@@ -12472,3 +12472,65 @@ func (s *integrationTestSuite) TestHostSoftwareWithTeamIdentifier() {
 	require.Equal(t, "/some/path/gh", getHostSoftwareResp.Software[2].InstalledVersions[0].InstalledPaths[0])
 	require.Nil(t, getHostSoftwareResp.Software[2].InstalledVersions[0].SignatureInformation)
 }
+
+func (s *integrationTestSuite) TestSecretVariables() {
+	t := s.T()
+	ctx := context.Background()
+
+	// Create the global GitOps user we'll use in tests.
+	u := &fleet.User{
+		Name:       "GitOps",
+		Email:      "gitops1@example.com",
+		GlobalRole: ptr.String(fleet.RoleGitOps),
+	}
+	require.NoError(t, u.SetPassword(test.GoodPassword, 10, 10))
+	_, err := s.ds.NewUser(ctx, u)
+	require.NoError(t, err)
+	s.setTokenForTest(t, "gitops1@example.com", test.GoodPassword)
+
+	// Empty request
+	req := secretVariablesRequest{}
+	var resp secretVariablesResponse
+	s.DoJSON("PUT", "/api/latest/fleet/spec/secret_variables", req, http.StatusOK, &resp)
+
+	// Secret variable name too long
+	req = secretVariablesRequest{
+		SecretVariables: []fleet.SecretVariable{
+			{
+				Name:  strings.Repeat("a", 256),
+				Value: "value",
+			},
+		},
+	}
+	httpResp := s.Do("PUT", "/api/latest/fleet/spec/secret_variables", req, http.StatusUnprocessableEntity)
+	assertBodyContains(t, httpResp, "secret variable name is too long")
+
+	// Secret variable name empty
+	req = secretVariablesRequest{
+		SecretVariables: []fleet.SecretVariable{
+			{
+				Name:  "  ",
+				Value: "value",
+			},
+		},
+	}
+	httpResp = s.Do("PUT", "/api/latest/fleet/spec/secret_variables", req, http.StatusUnprocessableEntity)
+	assertBodyContains(t, httpResp, "secret variable name cannot be empty")
+
+	validName := strings.Repeat("g", 255)
+	req = secretVariablesRequest{
+		SecretVariables: []fleet.SecretVariable{
+			{
+				Name:  "FLEET_SECRET_" + validName,
+				Value: "value",
+			},
+		},
+	}
+	s.DoJSON("PUT", "/api/latest/fleet/spec/secret_variables", req, http.StatusOK, &resp)
+
+	secrets, err := s.ds.GetSecretVariables(ctx, []string{validName})
+	require.NoError(t, err)
+	require.Len(t, secrets, 1)
+	assert.Equal(t, "value", secrets[0].Value)
+
+}

server/service/integration_enterprise_test.go

@@ -6089,15 +6089,6 @@ func (s *integrationEnterpriseTestSuite) TestGitOpsUserActions() {
 	}, http.StatusForbidden, &countTargetsResponse{})
 }
 
-func (s *integrationEnterpriseTestSuite) setTokenForTest(t *testing.T, email, password string) {
-	oldToken := s.token
-	t.Cleanup(func() {
-		s.token = oldToken
-	})
-
-	s.token = s.getCachedUserToken(email, password)
-}
-
 func (s *integrationEnterpriseTestSuite) TestDesktopEndpointWithInvalidPolicy() {
 	t := s.T()
 

server/service/secret_variables.go

@@ -0,0 +1,74 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
+	"github.com/fleetdm/fleet/v4/server/fleet"
+)
+
+const (
+	SecretVariablePrefix = "FLEET_SECRET_"
+	SecretVariableMaxLen = 255
+)
+
+// //////////////////////////////////////////////////////////////////////////////
+// Secret variables
+// //////////////////////////////////////////////////////////////////////////////
+
+type secretVariablesRequest struct {
+	SecretVariables []fleet.SecretVariable `json:"secrets"`
+}
+
+type secretVariablesResponse struct {
+	Err error `json:"error,omitempty"`
+}
+
+func (r secretVariablesResponse) error() error { return r.Err }
+
+func secretVariablesEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) {
+	req := request.(*secretVariablesRequest)
+	err := svc.CreateSecretVariables(ctx, req.SecretVariables)
+	return secretVariablesResponse{Err: err}, nil
+}
+
+func (svc *Service) CreateSecretVariables(ctx context.Context, secretVariables []fleet.SecretVariable) error {
+	// Do authorization check first so that we don't have to worry about it later in the flow.
+	if err := svc.authz.Authorize(ctx, &fleet.SecretVariable{}, fleet.ActionWrite); err != nil {
+		return err
+	}
+
+	privateKey := svc.config.Server.PrivateKey
+	if testSetEmptyPrivateKey {
+		privateKey = ""
+	}
+
+	if len(privateKey) == 0 {
+		return ctxerr.Wrap(ctx,
+			&fleet.BadRequestError{Message: "Couldn't save secret variables. Missing required private key. Learn how to configure the private key here: https://fleetdm.com/learn-more-about/fleet-server-private-key"})
+	}
+
+	// Preprocess: strip FLEET_SECRET_ prefix from variable names
+	for i, secretVariable := range secretVariables {
+		secretVariables[i].Name = fleet.Preprocess(strings.TrimPrefix(secretVariable.Name, SecretVariablePrefix))
+	}
+
+	// Validation
+	for _, secretVariable := range secretVariables {
+		if len(secretVariable.Name) == 0 {
+			return ctxerr.Wrap(ctx,
+				fleet.NewInvalidArgumentError("name", "secret variable name cannot be empty"))
+		}
+		if len(secretVariable.Name) > SecretVariableMaxLen {
+			return ctxerr.Wrap(ctx,
+				fleet.NewInvalidArgumentError("name", fmt.Sprintf("secret variable name is too long: %s", secretVariable.Name)))
+		}
+	}
+
+	if err := svc.ds.UpsertSecretVariables(ctx, secretVariables); err != nil {
+		return ctxerr.Wrap(ctx, err, "saving secret variables")
+	}
+	return nil
+}

server/service/secret_variables_test.go

@@ -0,0 +1,108 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/fleetdm/fleet/v4/server/contexts/viewer"
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/fleetdm/fleet/v4/server/mock"
+	"github.com/fleetdm/fleet/v4/server/ptr"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCreateSecretVariables(t *testing.T) {
+	t.Parallel()
+	ds := new(mock.Store)
+	svc, ctx := newTestService(t, ds, nil, nil)
+
+	ds.UpsertSecretVariablesFunc = func(ctx context.Context, secrets []fleet.SecretVariable) error {
+		return nil
+	}
+
+	t.Run("authorization checks", func(t *testing.T) {
+		testCases := []struct {
+			name       string
+			user       *fleet.User
+			shouldFail bool
+		}{
+			{
+				name:       "global admin",
+				user:       &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)},
+				shouldFail: false,
+			},
+			{
+				name:       "global maintainer",
+				user:       &fleet.User{GlobalRole: ptr.String(fleet.RoleMaintainer)},
+				shouldFail: false,
+			},
+			{
+				name:       "global gitops",
+				user:       &fleet.User{GlobalRole: ptr.String(fleet.RoleGitOps)},
+				shouldFail: false,
+			},
+			{
+				name:       "global observer",
+				user:       &fleet.User{GlobalRole: ptr.String(fleet.RoleObserver)},
+				shouldFail: true,
+			},
+			{
+				name:       "global observer+",
+				user:       &fleet.User{GlobalRole: ptr.String(fleet.RoleObserverPlus)},
+				shouldFail: true,
+			},
+			{
+				name:       "team admin",
+				user:       &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleAdmin}}},
+				shouldFail: true,
+			},
+			{
+				name:       "team maintainer",
+				user:       &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleMaintainer}}},
+				shouldFail: true,
+			},
+			{
+				name:       "team observer",
+				user:       &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleObserver}}},
+				shouldFail: true,
+			},
+			{
+				name:       "team observer+",
+				user:       &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleObserverPlus}}},
+				shouldFail: true,
+			},
+			{
+				name:       "team gitops",
+				user:       &fleet.User{Teams: []fleet.UserTeam{{Team: fleet.Team{ID: 1}, Role: fleet.RoleGitOps}}},
+				shouldFail: true,
+			},
+		}
+		for _, tt := range testCases {
+			t.Run(tt.name, func(t *testing.T) {
+				ctx = viewer.NewContext(ctx, viewer.Viewer{User: tt.user})
+
+				err := svc.CreateSecretVariables(ctx, []fleet.SecretVariable{{Name: "foo", Value: "bar"}})
+				checkAuthErr(t, tt.shouldFail, err)
+			})
+		}
+	})
+
+	t.Run("failure test", func(t *testing.T) {
+		ctx = viewer.NewContext(ctx, viewer.Viewer{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleGitOps)}})
+		testSetEmptyPrivateKey = true
+		t.Cleanup(func() {
+			testSetEmptyPrivateKey = false
+		})
+		err := svc.CreateSecretVariables(ctx, []fleet.SecretVariable{{Name: "foo", Value: "bar"}})
+		assert.ErrorContains(t, err, "Couldn't save secret variables. Missing required private key")
+		testSetEmptyPrivateKey = false
+
+		ds.UpsertSecretVariablesFunc = func(ctx context.Context, secrets []fleet.SecretVariable) error {
+			return errors.New("test error")
+		}
+		err = svc.CreateSecretVariables(ctx, []fleet.SecretVariable{{Name: "foo", Value: "bar"}})
+		assert.ErrorContains(t, err, "test error")
+	})
+
+}

server/service/testing_client.go

@@ -326,6 +326,15 @@ func (ts *withServer) getTestAdminToken() string {
 	return ts.cachedAdminToken
 }
 
+func (ts *withServer) setTokenForTest(t *testing.T, email, password string) {
+	oldToken := ts.token
+	t.Cleanup(func() {
+		ts.token = oldToken
+	})
+
+	ts.token = ts.getCachedUserToken(email, password)
+}
+
 // getCachedUserToken returns the cached auth token for the given test user email.
 // If it's not found, then a login request is performed and the token cached.
 func (ts *withServer) getCachedUserToken(email, password string) string {