add new changes for BYOD and fix issues (#22079)

for #21019 

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

<!-- Note that API documentation changes are now addressed by the
product design team. -->

- [x] Added/updated tests
- [x] Manual QA for all new/changed functionality

frontend/pages/hosts/details/DeviceUserPage/ManualEnrollMdmModal/ManualEnrollMdmModal.tsx

@@ -51,8 +51,9 @@ const ManualEnrollMdmModal = ({
             profile.
           </li>
           <li>
-            Select <b>Enroll</b> then enter your password.
+            Select <b>Install...</b> then confirm again clicking <b>Install</b>.
           </li>
+          <li>Enter your password when you get a prompt.</li>
           <li>
             Select <b>Done</b> to close this window and select <b>Refetch</b> on
             your My device page to tell <br /> your organization that MDM is on.

pkg/mdm/mdmtest/apple.go

@@ -217,7 +217,7 @@ func (c *TestAppleMDMClient) Enroll() error {
 }
 
 func (c *TestAppleMDMClient) fetchEnrollmentProfileFromDesktopURL() error {
-	return c.fetchEnrollmentProfile(
+	return c.fetchOTAProfile(
 		"/api/latest/fleet/device/" + c.desktopURLToken + "/mdm/apple/manual_enrollment_profile",
 	)
 }
@@ -229,6 +229,53 @@ func (c *TestAppleMDMClient) fetchEnrollmentProfileFromDEPURL() error {
 }
 
 func (c *TestAppleMDMClient) fetchEnrollmentProfileFromOTAURL() error {
+	return c.fetchOTAProfile(
+		"/api/latest/fleet/enrollment_profiles/ota?enroll_secret=" + url.QueryEscape(c.otaEnrollSecret),
+	)
+}
+
+func (c *TestAppleMDMClient) fetchOTAProfile(url string) error {
+	request, err := http.NewRequest("GET", c.fleetServerURL+url, nil)
+	if err != nil {
+		return fmt.Errorf("create request: %w", err)
+	}
+	// #nosec (this client is used for testing only)
+	cc := fleethttp.NewClient(fleethttp.WithTLSClientConfig(&tls.Config{
+		InsecureSkipVerify: true,
+	}))
+	response, err := cc.Do(request)
+	if err != nil {
+		return fmt.Errorf("send request: %w", err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("request error: %d, %s", response.StatusCode, response.Status)
+	}
+
+	body, err := io.ReadAll(response.Body)
+	if err != nil {
+		return fmt.Errorf("read body: %w", err)
+	}
+
+	p7, err := pkcs7.Parse(body)
+	if err != nil {
+		return fmt.Errorf("OTA profile is not XML nor PKCS7 parseable: %w", err)
+	}
+	err = p7.Verify()
+	if err != nil {
+		return fmt.Errorf("verifying OTA profile: %w", err)
+	}
+
+	var otaEnrollmentProfile struct {
+		PayloadContent struct {
+			URL string `plist:"URL"`
+		} `plist:"PayloadContent"`
+	}
+	err = plist.Unmarshal(p7.Content, &otaEnrollmentProfile)
+	if err != nil {
+		return fmt.Errorf("unmarshaling OTA enrollment response: %w", err)
+	}
+
 	rawDeviceInfo := []byte(fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
@@ -260,7 +307,7 @@ func (c *TestAppleMDMClient) fetchEnrollmentProfileFromOTAURL() error {
 
 		request, err := http.NewRequest(
 			"POST",
-			c.fleetServerURL+"/api/latest/fleet/ota_enrollment?enroll_secret="+c.otaEnrollSecret,
+			otaEnrollmentProfile.PayloadContent.URL,
 			bytes.NewReader(sig),
 		)
 		if err != nil {
@@ -297,7 +344,7 @@ func (c *TestAppleMDMClient) fetchEnrollmentProfileFromOTAURL() error {
 	if err != nil {
 		return fmt.Errorf("creating mock certificates: %w", err)
 	}
-	body, err := do(mockedCert, mockedKey)
+	body, err = do(mockedCert, mockedKey)
 	if err != nil {
 		return fmt.Errorf("first OTA request: %w", err)
 	}
@@ -326,7 +373,7 @@ func (c *TestAppleMDMClient) fetchEnrollmentProfileFromOTAURL() error {
 	if err != nil {
 		return fmt.Errorf("seconde OTA request: %w", err)
 	}
-	p7, err := pkcs7.Parse(body)
+	p7, err = pkcs7.Parse(body)
 	if err != nil {
 		return fmt.Errorf("enrollment profile is not XML nor PKCS7 parseable: %w", err)
 	}

server/datastore/mysql/apple_mdm.go

@@ -917,6 +917,7 @@ func createHostFromMDMDB(
 	tx sqlx.ExtContext,
 	logger log.Logger,
 	devices []hostToCreateFromMDM,
+	fromADE bool,
 	macOSTeam, iosTeam, ipadTeam *uint,
 ) (int64, []fleet.Host, error) {
 	// NOTE: order of arguments for teams is important, see statement.
@@ -981,6 +982,7 @@ func createHostFromMDMDB(
 				h.platform,
 				h.hardware_model,
 				h.hardware_serial,
+				h.hostname,
 				COALESCE(hmdm.enrolled, 0) as enrolled
 			FROM hosts h
 			LEFT JOIN host_mdm hmdm ON hmdm.host_id = h.id
@@ -1024,7 +1026,7 @@ func createHostFromMDMDB(
 		ctx,
 		tx,
 		appCfg.ServerSettings,
-		true,
+		fromADE,
 		unmanagedHostIDs...,
 	); err != nil {
 		return 0, nil, ctxerr.Wrap(ctx, err, "ingest mdm apple host upsert MDM info")
@@ -1046,7 +1048,7 @@ func (ds *Datastore) IngestMDMAppleDeviceFromOTAEnrollment(
 				HardwareModel:  deviceInfo.Product,
 			},
 		}
-		_, _, err := createHostFromMDMDB(ctx, tx, ds.logger, toInsert, teamID, teamID, teamID)
+		_, _, err := createHostFromMDMDB(ctx, tx, ds.logger, toInsert, false, teamID, teamID, teamID)
 		return ctxerr.Wrap(ctx, err, "creating host from OTA enrollment")
 	})
 }
@@ -1105,6 +1107,7 @@ func (ds *Datastore) IngestMDMAppleDevicesFromDEPSync(
 			tx,
 			ds.logger,
 			htc,
+			true,
 			teamIDs[0], teamIDs[1], teamIDs[2],
 		)
 		if err != nil {

server/fleet/errors.go

@@ -279,6 +279,36 @@ func (e PermissionError) PermissionError() []map[string]string {
 	return forbidden
 }
 
+// OTAForbiddenError is a special kind of forbidden error that intentionally
+// exposes information about the error so it can be shown in iPad/iPhone native
+// dialogs during OTA enrollment.
+//
+// I couldn't find any documentation but the way it works is:
+//
+// - if the response has a status code 403
+// - and the body has a `message` field
+//
+// the content of `message` will be displayed to the end user.
+type OTAForbiddenError struct {
+	ErrorWithUUID
+	InternalErr error
+}
+
+func (e OTAForbiddenError) Error() string {
+	return "Couldn't install the profile. Invalid enroll secret. Please contact your IT admin."
+}
+
+func (e OTAForbiddenError) StatusCode() int {
+	return http.StatusForbidden
+}
+
+func (e OTAForbiddenError) Internal() string {
+	if e.InternalErr == nil {
+		return ""
+	}
+	return e.InternalErr.Error()
+}
+
 // licenseError is returned when the application is not properly licensed.
 type licenseError struct {
 	ErrorWithUUID

server/mdm/apple/mobileconfig/profiles.go

@@ -1,9 +1,6 @@
 package mobileconfig
 
 import (
-	"encoding/xml"
-	"fmt"
-	"strings"
 	"text/template"
 )
 
@@ -119,13 +116,7 @@ var FleetCARootTemplate = template.Must(template.New("").Option("missingkey=erro
 </plist>
 `))
 
-var OTAMobileConfigTemplate = template.Must(template.New("").Funcs(template.FuncMap{"xml": func(v string) (string, error) {
-	var escaped strings.Builder
-	if err := xml.EscapeText(&escaped, []byte(v)); err != nil {
-		return "", fmt.Errorf("XML escaping in OTA profile: %w", err)
-	}
-	return escaped.String(), nil
-}}).Option("missingkey=error").Parse(`<?xml version="1.0" encoding="UTF-8"?>
+var OTAMobileConfigTemplate = template.Must(template.New("").Funcs(funcMap).Option("missingkey=error").Parse(`<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple Inc//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
   <dict>

server/service/apple_mdm.go

@@ -4204,8 +4204,8 @@ type mdmAppleOTARequest struct {
 func (mdmAppleOTARequest) DecodeRequest(ctx context.Context, r *http.Request) (interface{}, error) {
 	enrollSecret := r.URL.Query().Get("enroll_secret")
 	if enrollSecret == "" {
-		return nil, &fleet.BadRequestError{
-			Message: "enroll_secret query parameter is required",
+		return nil, &fleet.OTAForbiddenError{
+			InternalErr: errors.New("enroll_secret query parameter was empty"),
 		}
 	}
 
@@ -4289,7 +4289,9 @@ func (svc *Service) MDMAppleProcessOTAEnrollment(
 	enrollSecretInfo, err := svc.ds.VerifyEnrollSecret(ctx, enrollSecret)
 	if err != nil {
 		if fleet.IsNotFound(err) {
-			return nil, authz.ForbiddenWithInternal("invalid enroll secret provided", nil, nil, nil)
+			return nil, &fleet.OTAForbiddenError{
+				InternalErr: err,
+			}
 		}
 
 		return nil, ctxerr.Wrap(ctx, err, "validating enroll secret")

server/service/apple_mdm_test.go

@@ -234,6 +234,19 @@ func setupAppleMDMService(t *testing.T, license *fleet.LicenseInfo) (fleet.Servi
 func TestAppleMDMAuthorization(t *testing.T) {
 	svc, ctx, ds := setupAppleMDMService(t, &fleet.LicenseInfo{Tier: fleet.TierPremium})
 
+	ds.GetEnrollSecretsFunc = func(ctx context.Context, teamID *uint) ([]*fleet.EnrollSecret, error) {
+		return []*fleet.EnrollSecret{
+			{
+				Secret: "abcd",
+				TeamID: nil,
+			},
+			{
+				Secret: "efgh",
+				TeamID: nil,
+			},
+		}, nil
+	}
+
 	checkAuthErr := func(t *testing.T, err error, shouldFailWithAuth bool) {
 		t.Helper()
 

server/service/devices.go

@@ -3,8 +3,9 @@ package service
 import (
 	"context"
 	"crypto/x509"
+	"database/sql"
 	"encoding/json"
-	"fmt"
+	"errors"
 	"io"
 	"net/http"
 	"net/url"
@@ -529,34 +530,37 @@ func (svc *Service) GetDeviceMDMAppleEnrollmentProfile(ctx context.Context) ([]b
 		return nil, ctxerr.Wrap(ctx, fleet.NewPermissionError("forbidden: only device-authenticated hosts can access this endpoint"))
 	}
 
-	appConfig, err := svc.ds.AppConfig(ctx)
+	cfg, err := svc.ds.AppConfig(ctx)
 	if err != nil {
-		return nil, ctxerr.Wrap(ctx, err)
+		return nil, ctxerr.Wrap(ctx, err, "fetching app config")
 	}
 
-	topic, err := svc.mdmPushCertTopic(ctx)
-	if err != nil {
-		return nil, ctxerr.Wrap(ctx, err, "extracting topic from APNs cert")
+	host, ok := hostctx.FromContext(ctx)
+	if !ok {
+		return nil, ctxerr.Wrap(ctx, fleet.NewAuthRequiredError("internal error: missing host from request context"))
 	}
 
-	assets, err := svc.ds.GetAllMDMConfigAssetsByName(ctx, []fleet.MDMAssetName{
-		fleet.MDMAssetSCEPChallenge,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("loading SCEP challenge from the database: %w", err)
+	tmSecrets, err := svc.ds.GetEnrollSecrets(ctx, host.TeamID)
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		return nil, ctxerr.Wrap(ctx, err, "getting host team enroll secrets")
+	}
+	if len(tmSecrets) == 0 && host.TeamID != nil {
+		tmSecrets, err = svc.ds.GetEnrollSecrets(ctx, nil)
+		if err != nil && !errors.Is(err, sql.ErrNoRows) {
+			return nil, ctxerr.Wrap(ctx, err, "getting no team enroll secrets")
+		}
+	}
+	if len(tmSecrets) == 0 {
+		return nil, &fleet.BadRequestError{Message: "unable to find an enroll secret to generate enrollment profile"}
 	}
 
-	enrollmentProf, err := apple_mdm.GenerateEnrollmentProfileMobileconfig(
-		appConfig.OrgInfo.OrgName,
-		appConfig.ServerSettings.ServerURL,
-		string(assets[fleet.MDMAssetSCEPChallenge].Value),
-		topic,
-	)
+	enrollSecret := tmSecrets[0].Secret
+	profBytes, err := apple_mdm.GenerateOTAEnrollmentProfileMobileconfig(cfg.OrgInfo.OrgName, cfg.ServerSettings.ServerURL, enrollSecret)
 	if err != nil {
-		return nil, ctxerr.Wrap(ctx, err, "generating manual enrollment profile")
+		return nil, ctxerr.Wrap(ctx, err, "generating ota mobileconfig file for manual enrollment")
 	}
 
-	signed, err := mdmcrypto.Sign(ctx, enrollmentProf, svc.ds)
+	signed, err := mdmcrypto.Sign(ctx, profBytes, svc.ds)
 	if err != nil {
 		return nil, ctxerr.Wrap(ctx, err, "signing profile")
 	}

server/service/integration_mdm_profiles_test.go

@@ -4198,10 +4198,6 @@ func (s *integrationMDMTestSuite) TestBatchSetMDMProfilesBackwardsCompat() {
 	)
 }
 
-func (s *integrationMDMTestSuite) TestGetManualEnrollmentProfile() {
-	s.downloadAndVerifyEnrollmentProfile("/api/latest/fleet/enrollment_profiles/manual")
-}
-
 func (s *integrationMDMTestSuite) TestMDMBatchSetProfilesKeepsReservedNames() {
 	t := s.T()
 	ctx := context.Background()

server/service/integration_mdm_test.go

@@ -556,6 +556,14 @@ func (s *integrationMDMTestSuite) SetupSuite() {
 	// enable MDM flows
 	s.appleCoreCertsSetup()
 
+	// create a global enroll secret
+	var applyResp applyEnrollSecretSpecResponse
+	s.DoJSON("POST", "/api/latest/fleet/spec/enroll_secret", applyEnrollSecretSpecRequest{
+		Spec: &fleet.EnrollSecretSpec{
+			Secrets: []*fleet.EnrollSecret{{Secret: "global-secret"}},
+		},
+	}, http.StatusOK, &applyResp)
+
 	s.T().Cleanup(fleetdmSrv.Close)
 	s.T().Cleanup(s.appleVPPConfigSrv.Close)
 	s.T().Cleanup(s.appleITunesSrv.Close)
@@ -1045,6 +1053,7 @@ func createHostThenEnrollMDM(ds fleet.Datastore, fleetServerURL string, t *testi
 		NodeKey:         ptr.String(t.Name() + uuid.New().String()),
 		Hostname:        fmt.Sprintf("%sfoo.local", t.Name()),
 		Platform:        "darwin",
+		HardwareModel:   "MacBookPro16,1",
 
 		UUID:           mdmDevice.UUID,
 		HardwareSerial: mdmDevice.SerialNumber,
@@ -1127,7 +1136,7 @@ func (s *integrationMDMTestSuite) TestDeviceMDMManualEnroll() {
 	s.DoRaw("GET", "/api/latest/fleet/device/invalid_token/mdm/apple/manual_enrollment_profile", nil, http.StatusUnauthorized)
 
 	// valid token downloads the profile
-	s.downloadAndVerifyEnrollmentProfile("/api/latest/fleet/device/" + token + "/mdm/apple/manual_enrollment_profile")
+	s.downloadAndVerifyOTAEnrollmentProfile("/api/latest/fleet/device/" + token + "/mdm/apple/manual_enrollment_profile")
 }
 
 func (s *integrationMDMTestSuite) TestAppleMDMDeviceEnrollment() {
@@ -5497,6 +5506,45 @@ func (s *integrationMDMTestSuite) downloadAndVerifyEnrollmentProfile(path string
 	return s.verifyEnrollmentProfile(body, "")
 }
 
+func (s *integrationMDMTestSuite) downloadAndVerifyOTAEnrollmentProfile(path string) {
+	t := s.T()
+
+	resp := s.DoRaw("GET", path, nil, http.StatusOK)
+	rawProfile, err := io.ReadAll(resp.Body)
+	resp.Body.Close()
+	require.NoError(t, err)
+	require.Contains(t, resp.Header, "Content-Disposition")
+	require.Contains(t, resp.Header, "Content-Type")
+	require.Contains(t, resp.Header, "X-Content-Type-Options")
+	require.Contains(t, resp.Header.Get("Content-Disposition"), "attachment;")
+	require.Contains(t, resp.Header.Get("Content-Type"), "application/x-apple-aspen-config")
+	require.Contains(t, resp.Header.Get("X-Content-Type-Options"), "nosniff")
+	headerLen, err := strconv.Atoi(resp.Header.Get("Content-Length"))
+	require.NoError(t, err)
+	require.Equal(t, len(rawProfile), headerLen)
+
+	p7, err := pkcs7.Parse(rawProfile)
+	require.NoError(t, err)
+	rootCA := x509.NewCertPool()
+
+	assets, err := s.ds.GetAllMDMConfigAssetsByName(context.Background(), []fleet.MDMAssetName{
+		fleet.MDMAssetCACert,
+	})
+	require.NoError(t, err)
+
+	require.True(t, rootCA.AppendCertsFromPEM(assets[fleet.MDMAssetCACert].Value))
+	require.NoError(t, p7.VerifyWithChain(rootCA))
+
+	var otaEnrollmentProfile struct {
+		PayloadContent struct {
+			URL string `plist:"URL"`
+		} `plist:"PayloadContent"`
+	}
+	err = plist.Unmarshal(p7.Content, &otaEnrollmentProfile)
+	require.NoError(t, err)
+	require.Contains(t, otaEnrollmentProfile.PayloadContent.URL, s.getConfig().ServerSettings.ServerURL+"/api/v1/fleet/ota_enrollment")
+}
+
 func (s *integrationMDMTestSuite) verifyEnrollmentProfile(rawProfile []byte, enrollmentRef string) *enrollmentProfile {
 	t := s.T()
 	var profile enrollmentProfile
@@ -9828,7 +9876,10 @@ func (s *integrationMDMTestSuite) TestAPNsPushCron() {
 	defer func() { s.pushProvider.PushFunc = originalPushMock }()
 
 	var recordedPushes []*mdm.Push
+	var mu sync.Mutex
 	s.pushProvider.PushFunc = func(pushes []*mdm.Push) (map[string]*push.Response, error) {
+		mu.Lock()
+		defer mu.Unlock()
 		recordedPushes = pushes
 		return mockSuccessfulPush(pushes)
 	}
@@ -11149,7 +11200,7 @@ func (s *integrationMDMTestSuite) TestEnrollmentProfilesWithSpecialChars() {
 	// manual enrollment from My Device
 	token := "token_test_manual_enroll"
 	createHostAndDeviceToken(t, s.ds, token)
-	s.downloadAndVerifyEnrollmentProfile("/api/latest/fleet/device/" + token + "/mdm/apple/manual_enrollment_profile")
+	s.downloadAndVerifyOTAEnrollmentProfile("/api/latest/fleet/device/" + token + "/mdm/apple/manual_enrollment_profile")
 
 	// automatic enrollment by token
 	rawMsg := json.RawMessage(`{"allow_pairing": true}`)
@@ -11209,9 +11260,9 @@ func (s *integrationMDMTestSuite) TestOTAEnrollment() {
 </plist>`)
 
 	// request with no enroll secret
-	httpResp := s.DoRawNoAuth("POST", "/api/latest/fleet/ota_enrollment", reqBody, http.StatusBadRequest)
+	httpResp := s.DoRawNoAuth("POST", "/api/latest/fleet/ota_enrollment", reqBody, http.StatusForbidden)
 	errMsg := extractServerErrorText(httpResp.Body)
-	require.Contains(t, errMsg, "enroll_secret query parameter is required")
+	require.Contains(t, errMsg, "Couldn't install the profile. Invalid enroll secret. Please contact your IT admin.")
 	require.NoError(t, httpResp.Body.Close())
 
 	// request with no body
@@ -11237,14 +11288,14 @@ func (s *integrationMDMTestSuite) TestOTAEnrollment() {
 	// request with invalid apple signature
 	httpResp = s.DoRawNoAuth("POST", "/api/latest/fleet/ota_enrollment?enroll_secret=foo", signedReqBody, http.StatusForbidden)
 	errMsg = extractServerErrorText(httpResp.Body)
-	require.Contains(t, errMsg, "forbidden")
+	require.Contains(t, errMsg, "Couldn't install the profile. Invalid enroll secret. Please contact your IT admin.")
 	require.NoError(t, httpResp.Body.Close())
 
 	// request with invalid device signature
 	os.Setenv("FLEET_DEV_MDM_APPLE_DISABLE_DEVICE_INFO_CERT_VERIFY", "1")
 	httpResp = s.DoRawNoAuth("POST", "/api/latest/fleet/ota_enrollment?enroll_secret=foo", signedReqBody, http.StatusForbidden)
 	errMsg = extractServerErrorText(httpResp.Body)
-	require.Contains(t, errMsg, "forbidden")
+	require.Contains(t, errMsg, "Couldn't install the profile. Invalid enroll secret. Please contact your IT admin.")
 	require.NoError(t, httpResp.Body.Close())
 
 	// request without serial number