Add retries to MDM profile verification (#13811)

changes/11099-mdm-profiles-retries

@@ -0,0 +1,2 @@
+ - Updated MDM profile verification so that an install profile command will be retried once if the command
+ resulted in an error or if osquery cannot confirm that the expected profile is installed. 

server/datastore/mysql/apple_mdm.go

@@ -1593,22 +1593,59 @@ func (ds *Datastore) UpdateOrDeleteHostMDMAppleProfile(ctx context.Context, prof
 	return err
 }
 
-func (ds *Datastore) UpdateHostMDMProfilesVerification(ctx context.Context, host *fleet.Host, verified, failed []string) error {
+func (ds *Datastore) UpdateHostMDMProfilesVerification(ctx context.Context, hostUUID string, toVerify, toFail, toRetry []string) error {
 	return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
-		if err := setMDMProfilesVerifiedDB(ctx, tx, host, verified); err != nil {
+		if err := setMDMProfilesVerifiedDB(ctx, tx, hostUUID, toVerify); err != nil {
 			return err
 		}
-		if err := setMDMProfilesFailedDB(ctx, tx, host, failed); err != nil {
+		if err := setMDMProfilesFailedDB(ctx, tx, hostUUID, toFail); err != nil {
+			return err
+		}
+		if err := setMDMProfilesRetryDB(ctx, tx, hostUUID, toRetry); err != nil {
 			return err
 		}
 		return nil
 	})
 }
 
+// setMDMProfilesRetryDB sets the status of the given identifiers to retry (nil) and increments the retry count
+func setMDMProfilesRetryDB(ctx context.Context, tx sqlx.ExtContext, hostUUID string, identifiers []string) error {
+	if len(identifiers) == 0 {
+		return nil
+	}
+
+	stmt := `
+UPDATE
+	host_mdm_apple_profiles
+SET
+	status = NULL,
+	detail = '',
+	retries = retries + 1	
+WHERE		
+	host_uuid = ?
+	AND operation_type = ?	
+	AND profile_identifier IN(?)`
+
+	args := []interface{}{
+		hostUUID,
+		fleet.MDMAppleOperationTypeInstall,
+		identifiers,
+	}
+	stmt, args, err := sqlx.In(stmt, args...)
+	if err != nil {
+		return ctxerr.Wrap(ctx, err, "building sql statement to set retry host macOS profiles")
+	}
+
+	if _, err := tx.ExecContext(ctx, stmt, args...); err != nil {
+		return ctxerr.Wrap(ctx, err, "setting retry host macOS profiles")
+	}
+	return nil
+}
+
 // setMDMProfilesFailedDB sets the status of the given identifiers to failed if the current status
 // is verifying or verified. It also sets the detail to a message indicating that the profile was
 // either verifying or verified. Only profiles with the install operation type are updated.
-func setMDMProfilesFailedDB(ctx context.Context, tx sqlx.ExtContext, host *fleet.Host, identifiers []string) error {
+func setMDMProfilesFailedDB(ctx context.Context, tx sqlx.ExtContext, hostUUID string, identifiers []string) error {
 	if len(identifiers) == 0 {
 		return nil
 	}
@@ -1630,7 +1667,7 @@ WHERE
 		fleet.HostMDMProfileDetailFailedWasVerifying,
 		fleet.HostMDMProfileDetailFailedWasVerified,
 		fleet.MDMAppleDeliveryFailed,
-		host.UUID,
+		hostUUID,
 		[]interface{}{fleet.MDMAppleDeliveryVerifying, fleet.MDMAppleDeliveryVerified},
 		fleet.MDMAppleOperationTypeInstall,
 		identifiers,
@@ -1648,7 +1685,7 @@ WHERE
 
 // setMDMProfilesVerifiedDB sets the status of the given identifiers to verified if the current
 // status is verifying. Only profiles with the install operation type are updated.
-func setMDMProfilesVerifiedDB(ctx context.Context, tx sqlx.ExtContext, host *fleet.Host, identifiers []string) error {
+func setMDMProfilesVerifiedDB(ctx context.Context, tx sqlx.ExtContext, hostUUID string, identifiers []string) error {
 	if len(identifiers) == 0 {
 		return nil
 	}
@@ -1667,7 +1704,7 @@ WHERE
 
 	args := []interface{}{
 		fleet.MDMAppleDeliveryVerified,
-		host.UUID,
+		hostUUID,
 		[]interface{}{fleet.MDMAppleDeliveryVerifying, fleet.MDMAppleDeliveryFailed},
 		fleet.MDMAppleOperationTypeInstall,
 		identifiers,
@@ -1720,6 +1757,45 @@ WHERE
 	return byIdentifier, nil
 }
 
+func (ds *Datastore) GetHostMDMProfilesRetryCounts(ctx context.Context, hostUUID string) ([]fleet.HostMDMProfileRetryCount, error) {
+	stmt := `
+SELECT
+	profile_identifier,
+	retries
+FROM
+	host_mdm_apple_profiles hmap
+WHERE
+	hmap.host_uuid = ?`
+
+	var dest []fleet.HostMDMProfileRetryCount
+	if err := sqlx.SelectContext(ctx, ds.reader(ctx), &dest, stmt, hostUUID); err != nil {
+		return nil, ctxerr.Wrap(ctx, err, fmt.Sprintf("getting retry counts for host %s", hostUUID))
+	}
+
+	return dest, nil
+}
+
+func (ds *Datastore) GetHostMDMProfileRetryCountByCommandUUID(ctx context.Context, hostUUID, cmdUUID string) (fleet.HostMDMProfileRetryCount, error) {
+	stmt := `
+SELECT
+	profile_identifier, retries
+FROM
+	host_mdm_apple_profiles hmap
+WHERE
+	hmap.host_uuid = ?
+	AND hmap.command_uuid = ?`
+
+	var dest fleet.HostMDMProfileRetryCount
+	if err := sqlx.GetContext(ctx, ds.reader(ctx), &dest, stmt, hostUUID, cmdUUID); err != nil {
+		if err == sql.ErrNoRows {
+			return dest, notFound("HostMDMCommand").WithMessage(fmt.Sprintf("command uuid %s not found for host uuid %s", cmdUUID, hostUUID))
+		}
+		return dest, ctxerr.Wrap(ctx, err, fmt.Sprintf("getting retry count for host %s command uuid %s", hostUUID, cmdUUID))
+	}
+
+	return dest, nil
+}
+
 func subqueryHostsMacOSSettingsStatusFailing() (string, []interface{}) {
 	sql := `
             SELECT

server/datastore/mysql/apple_mdm_test.go

@@ -1449,6 +1449,10 @@ func upsertHostCPs(
 	upserts := []*fleet.MDMAppleBulkUpsertHostProfilePayload{}
 	for _, h := range hosts {
 		for _, cp := range profiles {
+			csum := []byte("csum")
+			if cp.Checksum != nil {
+				csum = cp.Checksum
+			}
 			payload := fleet.MDMAppleBulkUpsertHostProfilePayload{
 				ProfileID:         cp.ProfileID,
 				ProfileIdentifier: cp.Identifier,
@@ -1457,7 +1461,7 @@ func upsertHostCPs(
 				CommandUUID:       "",
 				OperationType:     opType,
 				Status:            status,
-				Checksum:          []byte("csum"),
+				Checksum:          csum,
 			}
 			upserts = append(upserts, &payload)
 		}
@@ -3927,6 +3931,15 @@ func testSetVerifiedMacOSProfiles(t *testing.T, ds *Datastore) {
 		}
 	}
 
+	adHocSetVerifying := func(hostUUID, profileIndentifier string) {
+		ExecAdhocSQL(t, ds, func(tx sqlx.ExtContext) error {
+			_, err := tx.ExecContext(ctx,
+				`UPDATE host_mdm_apple_profiles SET status = ? WHERE host_uuid = ? AND profile_identifier = ?`,
+				fleet.MDMAppleDeliveryVerifying, hostUUID, profileIndentifier)
+			return err
+		})
+	}
+
 	// initialize the host MDM profile statuses
 	upsertHostCPs(hosts, []*fleet.MDMAppleConfigProfile{storedByIdentifier[cp1.Identifier]}, fleet.MDMAppleOperationTypeInstall, &fleet.MDMAppleDeliveryPending, ctx, ds, t)
 	upsertHostCPs(hosts, []*fleet.MDMAppleConfigProfile{storedByIdentifier[cp2.Identifier]}, fleet.MDMAppleOperationTypeInstall, &fleet.MDMAppleDeliveryVerifying, ctx, ds, t)
@@ -4021,7 +4034,7 @@ func testSetVerifiedMacOSProfiles(t *testing.T, ds *Datastore) {
 		return err
 	})
 
-	// after the grace period, status changes to "failed" if a profile is missing (i.e. not installed)
+	// after the grace period and one retry attempt, status changes to "failed" if a profile is missing (i.e. not installed)
 	require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, hosts[2], profilesByIdentifier([]*fleet.HostMacOSProfile{
 		{
 			Identifier:  cp1.Identifier,
@@ -4034,10 +4047,27 @@ func testSetVerifiedMacOSProfiles(t *testing.T, ds *Datastore) {
 			InstallDate: time.Now(),
 		},
 	})))
-	expectedHostMDMStatus[hosts[2].ID][cp3.Identifier] = fleet.MDMAppleDeliveryFailed // cp3 is missing
+	expectedHostMDMStatus[hosts[2].ID][cp3.Identifier] = fleet.MDMAppleDeliveryPending // first retry for cp3
+	checkHostMDMProfileStatuses()
+	// simulate retry command acknowledged by setting status to "verifying"
+	adHocSetVerifying(hosts[2].UUID, cp3.Identifier)
+	// report osquery results again with cp3 still missing
+	require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, hosts[2], profilesByIdentifier([]*fleet.HostMacOSProfile{
+		{
+			Identifier:  cp1.Identifier,
+			DisplayName: cp1.Name,
+			InstallDate: time.Now(),
+		},
+		{
+			Identifier:  cp2.Identifier,
+			DisplayName: cp2.Name,
+			InstallDate: time.Now(),
+		},
+	})))
+	expectedHostMDMStatus[hosts[2].ID][cp3.Identifier] = fleet.MDMAppleDeliveryFailed // still missing after retry so expect cp3 to fail
 	checkHostMDMProfileStatuses()
 
-	// after the grace period, status changes to "failed" if a profile is outdated (i.e. installed
+	// after the grace period and one retry attempt, status changes to "failed" if a profile is outdated (i.e. installed
 	// before the updated at timestamp of the profile)
 	require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, hosts[2], profilesByIdentifier([]*fleet.HostMacOSProfile{
 		{
@@ -4051,7 +4081,24 @@ func testSetVerifiedMacOSProfiles(t *testing.T, ds *Datastore) {
 			InstallDate: time.Now().Add(-48 * time.Hour),
 		},
 	})))
-	expectedHostMDMStatus[hosts[2].ID][cp2.Identifier] = fleet.MDMAppleDeliveryFailed // cp2 is outdated
+	expectedHostMDMStatus[hosts[2].ID][cp2.Identifier] = fleet.MDMAppleDeliveryPending // first retry for cp2
+	checkHostMDMProfileStatuses()
+	// simulate retry command acknowledged by setting status to "verifying"
+	adHocSetVerifying(hosts[2].UUID, cp2.Identifier)
+	// report osquery results again with cp2 still outdated
+	require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, hosts[2], profilesByIdentifier([]*fleet.HostMacOSProfile{
+		{
+			Identifier:  cp1.Identifier,
+			DisplayName: cp1.Name,
+			InstallDate: time.Now(),
+		},
+		{
+			Identifier:  cp2.Identifier,
+			DisplayName: cp2.Name,
+			InstallDate: time.Now().Add(-48 * time.Hour),
+		},
+	})))
+	expectedHostMDMStatus[hosts[2].ID][cp2.Identifier] = fleet.MDMAppleDeliveryFailed // still outdated after retry so expect cp2 to fail
 	checkHostMDMProfileStatuses()
 }
 
@@ -4674,6 +4721,13 @@ func TestMDMProfileVerification(t *testing.T) {
 		})
 	}
 
+	setRetries := func(t *testing.T, hostUUID string, retries uint) {
+		ExecAdhocSQL(t, ds, func(tx sqlx.ExtContext) error {
+			_, err := tx.ExecContext(ctx, `UPDATE host_mdm_apple_profiles SET retries = ? WHERE host_uuid = ?`, retries, hostUUID)
+			return err
+		})
+	}
+
 	checkHostStatus := func(t *testing.T, h *fleet.Host, expectedStatus fleet.MDMAppleDeliveryStatus, expectedDetail string) error {
 		gotProfs, err := ds.GetHostMDMProfiles(ctx, h.UUID)
 		if err != nil {
@@ -4694,32 +4748,43 @@ func TestMDMProfileVerification(t *testing.T) {
 		return nil
 	}
 
-	t.Run("MissingProfile", func(t *testing.T) {
-		// missing profile, verifying and verified statuses should change to failed after the grace period
+	initializeProfile := func(t *testing.T, h *fleet.Host, cp *fleet.MDMAppleConfigProfile, status fleet.MDMAppleDeliveryStatus, prevRetries uint) {
+		upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &status, ctx, ds, t)
+		require.NoError(t, checkHostStatus(t, h, status, ""))
+		setRetries(t, h.UUID, prevRetries)
+	}
+
+	cleanupProfiles := func(t *testing.T) {
+		ExecAdhocSQL(t, ds, func(tx sqlx.ExtContext) error {
+			_, err := tx.ExecContext(ctx, `DELETE FROM mdm_apple_configuration_profiles; DELETE FROM host_mdm_apple_profiles`)
+			return err
+		})
+	}
+
+	t.Run("MissingProfileWithRetry", func(t *testing.T) {
+		defer cleanupProfiles(t)
+		// missing profile, verifying and verified statuses should change to failed after the grace
+		// period and one retry
 		cases := []testCase{
 			{
 				name:           "PendingThenMissing",
 				initialStatus:  fleet.MDMAppleDeliveryPending,
 				expectedStatus: fleet.MDMAppleDeliveryPending, // no change
-				expectedDetail: "",
 			},
 			{
 				name:           "VerifyingThenMissing",
 				initialStatus:  fleet.MDMAppleDeliveryVerifying,
 				expectedStatus: fleet.MDMAppleDeliveryFailed, // change to failed
-				expectedDetail: string(fleet.HostMDMProfileDetailFailedWasVerifying),
 			},
 			{
 				name:           "VerifiedThenMissing",
 				initialStatus:  fleet.MDMAppleDeliveryVerified,
 				expectedStatus: fleet.MDMAppleDeliveryFailed, // change to failed
-				expectedDetail: string(fleet.HostMDMProfileDetailFailedWasVerified),
 			},
 			{
 				name:           "FailedThenMissing",
 				initialStatus:  fleet.MDMAppleDeliveryFailed,
 				expectedStatus: fleet.MDMAppleDeliveryFailed, // no change
-				expectedDetail: "",
 			},
 		}
 
@@ -4730,8 +4795,7 @@ func TestMDMProfileVerification(t *testing.T) {
 			var reportedProfiles []*fleet.HostMacOSProfile // no profiles reported for this test
 
 			// initialize
-			upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t)
-			require.NoError(t, checkHostStatus(t, h, tc.initialStatus, ""))
+			initializeProfile(t, h, cp, tc.initialStatus, 0)
 
 			// within grace period
 			setProfileUpdatedAt(t, cp, twoMinutesAgo)
@@ -4739,13 +4803,24 @@ func TestMDMProfileVerification(t *testing.T) {
 			require.NoError(t, checkHostStatus(t, h, tc.initialStatus, "")) // if missing within grace period, no change
 
 			// reinitialize
-			upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t)
-			require.NoError(t, checkHostStatus(t, h, tc.initialStatus, ""))
+			initializeProfile(t, h, cp, tc.initialStatus, 0)
 
 			// outside grace period
 			setProfileUpdatedAt(t, cp, twoHoursAgo)
 			require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, h, profilesByIdentifier(reportedProfiles)))
-			require.NoError(t, checkHostStatus(t, h, tc.expectedStatus, tc.expectedDetail)) // grace period expired, check expected status
+			if tc.expectedStatus == fleet.MDMAppleDeliveryFailed {
+				// grace period expired, first failure gets retried so status should be pending and empty detail
+				require.NoError(t, checkHostStatus(t, h, fleet.MDMAppleDeliveryPending, ""), tc.name)
+			}
+
+			if tc.initialStatus != fleet.MDMAppleDeliveryPending {
+				// after retry, assume successful install profile command so status should be verifying
+				upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &fleet.MDMAppleDeliveryVerifying, ctx, ds, t)
+				// report osquery results
+				require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, h, profilesByIdentifier(reportedProfiles)))
+				// now we see the expected status
+				require.NoError(t, checkHostStatus(t, h, tc.expectedStatus, string(fleet.HostMDMProfileDetailFailedWasVerifying)), tc.name) // grace period expired, max retries so check expected status
+			}
 		}
 	})
 
@@ -4782,6 +4857,8 @@ func TestMDMProfileVerification(t *testing.T) {
 
 		for i, tc := range cases {
 			t.Run(tc.name, func(t *testing.T) {
+				defer cleanupProfiles(t)
+
 				// setup
 				h := test.NewHost(t, ds, tc.name, tc.name, tc.name, tc.name, twoMinutesAgo)
 				cp := setupTestProfile(t, fmt.Sprintf("%s-%d", tc.name, i))
@@ -4793,18 +4870,16 @@ func TestMDMProfileVerification(t *testing.T) {
 					},
 				}
 
-				// initialize
-				upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t)
-				require.NoError(t, checkHostStatus(t, h, tc.initialStatus, ""))
+				// initialize with no remaining retries
+				initializeProfile(t, h, cp, tc.initialStatus, 1)
 
 				// within grace period
 				setProfileUpdatedAt(t, cp, twoMinutesAgo)
 				require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, h, profilesByIdentifier(reportedProfiles)))
 				require.NoError(t, checkHostStatus(t, h, tc.initialStatus, "")) // outdated profiles are treated similar to missing profiles so status doesn't change if within grace period
 
-				// reinitalize
-				upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t)
-				require.NoError(t, checkHostStatus(t, h, tc.initialStatus, ""))
+				// reinitalize with no remaining retries
+				initializeProfile(t, h, cp, tc.initialStatus, 1)
 
 				// outside grace period
 				setProfileUpdatedAt(t, cp, twoHoursAgo)
@@ -4845,6 +4920,8 @@ func TestMDMProfileVerification(t *testing.T) {
 
 		for i, tc := range cases {
 			t.Run(tc.name, func(t *testing.T) {
+				defer cleanupProfiles(t)
+
 				// setup
 				h := test.NewHost(t, ds, tc.name, tc.name, tc.name, tc.name, twoMinutesAgo)
 				cp := setupTestProfile(t, fmt.Sprintf("%s-%d", tc.name, i))
@@ -4856,18 +4933,16 @@ func TestMDMProfileVerification(t *testing.T) {
 					},
 				}
 
-				// initialize
-				upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t)
-				require.NoError(t, checkHostStatus(t, h, tc.initialStatus, ""))
+				// initialize with no remaining retries
+				initializeProfile(t, h, cp, tc.initialStatus, 1)
 
 				// within grace period
 				setProfileUpdatedAt(t, cp, twoMinutesAgo)
 				require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, h, profilesByIdentifier(reportedProfiles)))
 				require.NoError(t, checkHostStatus(t, h, tc.expectedStatus, tc.expectedDetail)) // if found within grace period, verifying status can become verified so check expected status
 
-				// reinitialize
-				upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t)
-				require.NoError(t, checkHostStatus(t, h, tc.initialStatus, ""))
+				// reinitializewith no remaining retries
+				initializeProfile(t, h, cp, tc.initialStatus, 1)
 
 				// outside grace period
 				setProfileUpdatedAt(t, cp, twoHoursAgo)
@@ -4908,6 +4983,8 @@ func TestMDMProfileVerification(t *testing.T) {
 
 		for i, tc := range cases {
 			t.Run(tc.name, func(t *testing.T) {
+				defer cleanupProfiles(t)
+
 				// setup
 				h := test.NewHost(t, ds, tc.name, tc.name, tc.name, tc.name, twoMinutesAgo)
 				cp := setupTestProfile(t, fmt.Sprintf("%s-%d", tc.name, i))
@@ -4924,18 +5001,16 @@ func TestMDMProfileVerification(t *testing.T) {
 					},
 				}
 
-				// initialize
-				upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t)
-				require.NoError(t, checkHostStatus(t, h, tc.initialStatus, ""))
+				// initialize with no remaining retries
+				initializeProfile(t, h, cp, tc.initialStatus, 1)
 
 				// within grace period
 				setProfileUpdatedAt(t, cp, twoMinutesAgo)
 				require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, h, profilesByIdentifier(reportedProfiles)))
 				require.NoError(t, checkHostStatus(t, h, tc.expectedStatus, tc.expectedDetail)) // if found within grace period, verifying status can become verified so check expected status
 
-				// reinitialize
-				upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t)
-				require.NoError(t, checkHostStatus(t, h, tc.initialStatus, ""))
+				// reinitialize with no remaining retries
+				initializeProfile(t, h, cp, tc.initialStatus, 1)
 
 				// outside grace period
 				setProfileUpdatedAt(t, cp, twoHoursAgo)
@@ -4946,6 +5021,8 @@ func TestMDMProfileVerification(t *testing.T) {
 	})
 
 	t.Run("EarliestInstallDate", func(t *testing.T) {
+		defer cleanupProfiles(t)
+
 		hostString := "host-earliest-install-date"
 		h := test.NewHost(t, ds, hostString, hostString, hostString, hostString, twoMinutesAgo)
 
@@ -4967,27 +5044,24 @@ func TestMDMProfileVerification(t *testing.T) {
 		}
 		initialStatus := fleet.MDMAppleDeliveryVerifying
 
-		// initialize
-		upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{stored0}, fleet.MDMAppleOperationTypeInstall, &initialStatus, ctx, ds, t)
-		require.NoError(t, checkHostStatus(t, h, initialStatus, ""))
+		// initialize with no remaining retries
+		initializeProfile(t, h, stored0, initialStatus, 1)
 
 		// within grace period
 		setProfileUpdatedAt(t, stored0, twoMinutesAgo) // host is out of date but still within grace period
 		require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, h, profilesByIdentifier(reportedProfiles)))
 		require.NoError(t, checkHostStatus(t, h, fleet.MDMAppleDeliveryVerifying, "")) // no change
 
-		// reinitialize
-		upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{stored0}, fleet.MDMAppleOperationTypeInstall, &initialStatus, ctx, ds, t)
-		require.NoError(t, checkHostStatus(t, h, initialStatus, ""))
+		// reinitialize with no remaining retries
+		initializeProfile(t, h, stored0, initialStatus, 1)
 
 		// outside grace period
 		setProfileUpdatedAt(t, stored0, twoHoursAgo) // host is out of date and grace period has passed
 		require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, h, profilesByIdentifier(reportedProfiles)))
 		require.NoError(t, checkHostStatus(t, h, fleet.MDMAppleDeliveryFailed, string(fleet.HostMDMProfileDetailFailedWasVerifying))) // set to failed
 
-		// reinitialize
-		upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{stored0}, fleet.MDMAppleOperationTypeInstall, &initialStatus, ctx, ds, t)
-		require.NoError(t, checkHostStatus(t, h, initialStatus, ""))
+		// reinitialize with no remaining retries
+		initializeProfile(t, h, stored0, initialStatus, 1)
 
 		// save a copy of the config profile to team 1
 		cp.TeamID = ptr.Uint(1)

server/datastore/mysql/migrations/tables/20230911163618_AddRetriesColumnHostMdmAppleProfilesTable.go

@@ -0,0 +1,25 @@
+package tables
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20230911163618, Down_20230911163618)
+}
+
+func Up_20230911163618(tx *sql.Tx) error {
+	stmt := `
+ALTER TABLE host_mdm_apple_profiles
+	ADD COLUMN retries TINYINT(3) UNSIGNED NOT NULL DEFAULT 0`
+
+	if _, err := tx.Exec(stmt); err != nil {
+		return fmt.Errorf("add retries to host_mdm_apple_profiles: %w", err)
+	}
+	return nil
+}
+
+func Down_20230911163618(tx *sql.Tx) error {
+	return nil
+}

server/datastore/mysql/migrations/tables/20230911163618_AddRetriesColumnHostMdmAppleProfilesTable_test.go

@@ -0,0 +1,109 @@
+package tables
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUp_20230911163618(t *testing.T) {
+	db := applyUpToPrev(t)
+	insertStmt := `
+INSERT INTO  host_mdm_apple_profiles (
+	profile_id, 
+	profile_identifier, 
+	host_uuid, 
+	status, 
+	operation_type, 
+	detail, 
+	command_uuid, 
+	profile_name, 
+	checksum)
+VALUES
+	(?, ?, ?, ?, ?, ?, ?, ?, ?)`
+
+	args := []interface{}{
+		1,
+		"test-identifier",
+		"test-host-uuid",
+		fleet.MDMAppleDeliveryVerified,
+		fleet.MDMAppleOperationTypeInstall,
+		"test-detail",
+		"test-command-uuid",
+		"test-profile-name",
+		[]byte("test-checksum"),
+	}
+	execNoErr(t, db, insertStmt, args...)
+
+	applyNext(t, db)
+
+	// retrieve the stored value
+	var hmap struct {
+		ProfileID         uint                          `db:"profile_id"`
+		ProfileIdentifier string                        `db:"profile_identifier"`
+		HostUUID          string                        `db:"host_uuid"`
+		Status            *fleet.MDMAppleDeliveryStatus `db:"status"`
+		OperationType     fleet.MDMAppleOperationType   `db:"operation_type"`
+		Detail            string                        `db:"detail"`
+		CommandUUID       string                        `db:"command_uuid"`
+		ProfileName       string                        `db:"profile_name"`
+		Checksum          []byte                        `db:"checksum"`
+		Retries           uint                          `db:"retries"`
+	}
+
+	selectStmt := "SELECT * FROM host_mdm_apple_profiles WHERE host_uuid = ?"
+	require.NoError(t, db.Get(&hmap, selectStmt, "test-host-uuid"))
+	require.Equal(t, uint(1), hmap.ProfileID)
+	require.Equal(t, "test-identifier", hmap.ProfileIdentifier)
+	require.Equal(t, "test-host-uuid", hmap.HostUUID)
+	require.Equal(t, fleet.MDMAppleDeliveryVerified, *hmap.Status)
+	require.Equal(t, fleet.MDMAppleOperationTypeInstall, hmap.OperationType)
+	require.Equal(t, "test-detail", hmap.Detail)
+	require.Equal(t, "test-command-uuid", hmap.CommandUUID)
+	require.Equal(t, "test-profile-name", hmap.ProfileName)
+	require.True(t, bytes.HasPrefix(hmap.Checksum, []byte("test-checksum")))
+	require.Equal(t, uint(0), hmap.Retries)
+
+	insertStmt = `
+INSERT INTO  host_mdm_apple_profiles (
+	profile_id, 
+	profile_identifier, 
+	host_uuid, 
+	status, 
+	operation_type, 
+	detail, 
+	command_uuid, 
+	profile_name, 
+	checksum,
+	retries)
+VALUES
+	(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+
+	args = []interface{}{
+		1,
+		"test-identifier",
+		"test-host-uuid-2",
+		fleet.MDMAppleDeliveryVerified,
+		fleet.MDMAppleOperationTypeInstall,
+		"test-detail",
+		"test-command-uuid-2",
+		"test-profile-name",
+		[]byte("test-checksum"),
+		1,
+	}
+	execNoErr(t, db, insertStmt, args...)
+
+	require.NoError(t, db.Get(&hmap, selectStmt, "test-host-uuid-2"))
+	require.Equal(t, uint(1), hmap.ProfileID)
+	require.Equal(t, "test-identifier", hmap.ProfileIdentifier)
+	require.Equal(t, "test-host-uuid-2", hmap.HostUUID)
+	require.Equal(t, fleet.MDMAppleDeliveryVerified, *hmap.Status)
+	require.Equal(t, fleet.MDMAppleOperationTypeInstall, hmap.OperationType)
+	require.Equal(t, "test-detail", hmap.Detail)
+	require.Equal(t, "test-command-uuid-2", hmap.CommandUUID)
+	require.Equal(t, "test-profile-name", hmap.ProfileName)
+	require.True(t, bytes.HasPrefix(hmap.Checksum, []byte("test-checksum")))
+	require.Equal(t, uint(1), hmap.Retries)
+}

server/fleet/datastore.go

@@ -701,11 +701,20 @@ type Datastore interface {
 
 	SetDiskEncryptionResetStatus(ctx context.Context, hostID uint, status bool) error
 
-	// UpdateVerificationHostMacOSProfiles updates status of macOS profiles installed on a given host to verified.
-	UpdateHostMDMProfilesVerification(ctx context.Context, host *Host, verified, failed []string) error
+	// UpdateVerificationHostMacOSProfiles updates status of macOS profiles installed on a given
+	// host. The toVerify, toFail, and toRetry slices contain the identifiers of the profiles that
+	// should be verified, failed, and retried, respectively. For each profile in the toRetry slice,
+	// the retries count is incremented by 1 and the status is set to null so that an install
+	// profile command is enqueued the next time the profile manager cron runs.
+	UpdateHostMDMProfilesVerification(ctx context.Context, hostUUID string, toVerify, toFail, toRetry []string) error
 	// GetHostMDMProfilesExpected returns the expected MDM profiles for a given host. The map is
 	// keyed by the profile identifier.
 	GetHostMDMProfilesExpectedForVerification(ctx context.Context, host *Host) (map[string]*ExpectedMDMProfile, error)
+	// GetHostMDMProfilesRetryCounts returns a list of MDM profile retry counts for a given host.
+	GetHostMDMProfilesRetryCounts(ctx context.Context, hostUUID string) ([]HostMDMProfileRetryCount, error)
+	// GetHostMDMProfileRetryCountByCommandUUID returns the retry count for the specified
+	// host UUID and command UUID.
+	GetHostMDMProfileRetryCountByCommandUUID(ctx context.Context, hostUUID, cmdUUID string) (HostMDMProfileRetryCount, error)
 
 	// SetOrUpdateHostOrbitInfo inserts of updates the orbit info for a host
 	SetOrUpdateHostOrbitInfo(ctx context.Context, hostID uint, version string) error

server/fleet/mdm.go

@@ -128,3 +128,10 @@ func (ep ExpectedMDMProfile) IsWithinGracePeriod(hostDetailUpdatedAt time.Time)
 	gracePeriod := 1 * time.Hour
 	return hostDetailUpdatedAt.Before(ep.EarliestInstallDate.Add(gracePeriod))
 }
+
+// HostMDMProfileRetryCount represents the number of times Fleet has attempted to install
+// the identified profile on a host.
+type HostMDMProfileRetryCount struct {
+	ProfileIdentifier string `db:"profile_identifier"`
+	Retries           uint   `db:"retries"`
+}

server/mdm/apple/profile_verifier.go

@@ -6,11 +6,61 @@ import (
 	"github.com/fleetdm/fleet/v4/server/fleet"
 )
 
+// Profile verification is a set of related processes that run on the Fleet server to ensure that
+// the MDM profiles installed on a host are the ones expected by the Fleet server. Expected profiles
+// comprise the profiles that belong to the host's assigned team (or no
+// team, as applicable).
+//
+// The Fleet server enqueues commands to install profiles on hosts via the MDM
+// protocol. The Fleet server periodically runs a cron that enqueues install profile
+// commands for host profiles that do not have a verification status (i.e. status is null).
+// Install profile commands may be enqueued as a result of a variety of events, such as when a host
+// enrolls in Fleet, when a host's team membership changes, when a new profile is uploaded, when an
+// existing profile is modified, or when a failed profile is retried.
+//
+// Verification status of a host profile can change in the following ways:
+//
+// - When an install profile command is enqueued by the server, the verification status is set to "pending".
+//
+// - When the results of an install profile command are reported via the MDM protocol, the Fleet server
+//   parses the results and updates the host's verification status for the applicable profile. If the
+//   command was acknowledged, the verification status is set to "verifying". If the command resulted
+//   in an error, the server determines if the profile should be retried (in which case, a new install profile
+//   command will be enqueued by the server) or marked as "failed" and updates the datastore accordingly.
+//
+// - When host details are reported via osquery, the Fleet server ingests a list of installed
+//   profiles and compares the reported profiles with the list of profiles expected to be
+//   installed on the host. Expected profiles comprise the profiles that belong to the host's assigned
+//   team (or no  team, as applicable). If an expected profile is found, the verification status is
+//   set to "verified". If an expected profile is missing from the reported results, the server determines
+//   if the profile should be retried (in which case, a new install profile command will be enqueued by the server)
+//   or marked as "failed" and updates the datastore accordingly.
+
+// maxRetries is the maximum times an install profile command may be retried, after which marked as failed and no further
+// attempts will be made to install the profile.
+const maxRetries = 1
+
 // ProfileVerificationStore is the minimal interface required to get and update the verification
 // status of a host's MDM profiles. The Fleet Datastore satisfies this interface.
 type ProfileVerificationStore interface {
+	// GetHostMDMProfilesExpectedForVerification returns the expected MDM profiles for a given host. The map is
+	// keyed by the profile identifier.
 	GetHostMDMProfilesExpectedForVerification(ctx context.Context, host *fleet.Host) (map[string]*fleet.ExpectedMDMProfile, error)
-	UpdateHostMDMProfilesVerification(ctx context.Context, host *fleet.Host, verified, failed []string) error
+	// GetHostMDMProfilesRetryCounts returns the retry counts for the specified host.
+	GetHostMDMProfilesRetryCounts(ctx context.Context, hostUUID string) ([]fleet.HostMDMProfileRetryCount, error)
+	// GetHostMDMProfileRetryCountByCommandUUID returns the retry count for the specified
+	// host UUID and command UUID.
+	GetHostMDMProfileRetryCountByCommandUUID(ctx context.Context, hostUUID, commandUUID string) (fleet.HostMDMProfileRetryCount, error)
+	// UpdateHostMDMProfilesVerification updates status of macOS profiles installed on a given
+	// host. The toVerify, toFail, and toRetry slices contain the identifiers of the profiles that
+	// should be verified, failed, and retried, respectively. For each profile in the toRetry slice,
+	// the retries count is incremented by 1 and the status is set to null so that an install
+	// profile command is enqueued the next time the profile manager cron runs.
+	UpdateHostMDMProfilesVerification(ctx context.Context, hostUUID string, toVerify, toFail, toRetry []string) error
+	// UpdateOrDeleteHostMDMAppleProfile updates information about a single
+	// profile status. It deletes the row if the profile operation is "remove"
+	// and the status is "verifying" (i.e. successfully removed).
+	UpdateOrDeleteHostMDMAppleProfile(ctx context.Context, profile *fleet.HostMDMAppleProfile) error
 }
 
 var _ ProfileVerificationStore = (fleet.Datastore)(nil)
@@ -24,7 +74,7 @@ func VerifyHostMDMProfiles(ctx context.Context, ds ProfileVerificationStore, hos
 		return err
 	}
 
-	failed := make([]string, 0, len(expected))
+	missing := make([]string, 0, len(expected))
 	verified := make([]string, 0, len(expected))
 	for key, ep := range expected {
 		withinGracePeriod := ep.IsWithinGracePeriod(host.DetailUpdatedAt)
@@ -32,19 +82,71 @@ func VerifyHostMDMProfiles(ctx context.Context, ds ProfileVerificationStore, hos
 		if !ok {
 			// expected profile is missing from host
 			if !withinGracePeriod {
-				failed = append(failed, key)
+				missing = append(missing, key)
 			}
 			continue
 		}
 		if ip.InstallDate.Before(ep.EarliestInstallDate) {
 			// installed profile is outdated
 			if !withinGracePeriod {
-				failed = append(failed, key)
+				missing = append(missing, key)
 			}
 			continue
 		}
 		verified = append(verified, key)
 	}
 
-	return ds.UpdateHostMDMProfilesVerification(ctx, host, verified, failed)
+	toFail := make([]string, 0, len(missing))
+	toRetry := make([]string, 0, len(missing))
+	if len(missing) > 0 {
+		counts, err := ds.GetHostMDMProfilesRetryCounts(ctx, host.UUID)
+		if err != nil {
+			return err
+		}
+		retriesByProfileIdentifier := make(map[string]uint, len(counts))
+		for _, r := range counts {
+			retriesByProfileIdentifier[r.ProfileIdentifier] = r.Retries
+		}
+		for _, key := range missing {
+			if retriesByProfileIdentifier[key] < maxRetries {
+				// if we haven't hit the max retries, we set the host profile status to nil (which
+				// causes an install profile command to be enqueued the next time the profile
+				// manager cron runs) and increment the retry count
+				toRetry = append(toRetry, key)
+			} else {
+				// otherwise we set the host profile status to failed
+				toFail = append(toFail, key)
+			}
+		}
+	}
+
+	return ds.UpdateHostMDMProfilesVerification(ctx, host.UUID, verified, toFail, toRetry)
+}
+
+// HandleHostMDMProfileInstallResult ingests the result of an install profile command reported via
+// the MDM protocol and updates the verification status in the datastore. It is intended to be
+// called by the Fleet MDM checkin and command service install profile request handler.
+func HandleHostMDMProfileInstallResult(ctx context.Context, ds ProfileVerificationStore, hostUUID string, cmdUUID string, status *fleet.MDMAppleDeliveryStatus, detail string) error {
+	if status != nil && *status == fleet.MDMAppleDeliveryFailed {
+		m, err := ds.GetHostMDMProfileRetryCountByCommandUUID(ctx, hostUUID, cmdUUID)
+		if err != nil {
+			return err
+		}
+
+		if m.Retries < maxRetries {
+			// if we haven't hit the max retries, we set the host profile status to nil (which
+			// causes an install profile command to be enqueued the next time the profile
+			// manager cron runs) and increment the retry count
+			return ds.UpdateHostMDMProfilesVerification(ctx, hostUUID, nil, nil, []string{m.ProfileIdentifier})
+		}
+	}
+
+	// otherwise update status and detail as usual
+	return ds.UpdateOrDeleteHostMDMAppleProfile(ctx, &fleet.HostMDMAppleProfile{
+		CommandUUID:   cmdUUID,
+		HostUUID:      hostUUID,
+		Status:        status,
+		Detail:        detail,
+		OperationType: fleet.MDMAppleOperationTypeInstall,
+	})
 }

server/mock/datastore_mock.go

@@ -488,10 +488,14 @@ type GetHostDiskEncryptionKeyFunc func(ctx context.Context, hostID uint) (*fleet
 
 type SetDiskEncryptionResetStatusFunc func(ctx context.Context, hostID uint, status bool) error
 
-type UpdateHostMDMProfilesVerificationFunc func(ctx context.Context, host *fleet.Host, verified []string, failed []string) error
+type UpdateHostMDMProfilesVerificationFunc func(ctx context.Context, hostUUID string, toVerify []string, toFail []string, toRetry []string) error
 
 type GetHostMDMProfilesExpectedForVerificationFunc func(ctx context.Context, host *fleet.Host) (map[string]*fleet.ExpectedMDMProfile, error)
 
+type GetHostMDMProfilesRetryCountsFunc func(ctx context.Context, hostUUID string) ([]fleet.HostMDMProfileRetryCount, error)
+
+type GetHostMDMProfileRetryCountByCommandUUIDFunc func(ctx context.Context, hostUUID string, cmdUUID string) (fleet.HostMDMProfileRetryCount, error)
+
 type SetOrUpdateHostOrbitInfoFunc func(ctx context.Context, hostID uint, version string) error
 
 type ReplaceHostDeviceMappingFunc func(ctx context.Context, id uint, mappings []*fleet.HostDeviceMapping) error
@@ -1386,6 +1390,12 @@ type DataStore struct {
 	GetHostMDMProfilesExpectedForVerificationFunc        GetHostMDMProfilesExpectedForVerificationFunc
 	GetHostMDMProfilesExpectedForVerificationFuncInvoked bool
 
+	GetHostMDMProfilesRetryCountsFunc        GetHostMDMProfilesRetryCountsFunc
+	GetHostMDMProfilesRetryCountsFuncInvoked bool
+
+	GetHostMDMProfileRetryCountByCommandUUIDFunc        GetHostMDMProfileRetryCountByCommandUUIDFunc
+	GetHostMDMProfileRetryCountByCommandUUIDFuncInvoked bool
+
 	SetOrUpdateHostOrbitInfoFunc        SetOrUpdateHostOrbitInfoFunc
 	SetOrUpdateHostOrbitInfoFuncInvoked bool
 
@@ -3307,11 +3317,11 @@ func (s *DataStore) SetDiskEncryptionResetStatus(ctx context.Context, hostID uin
 	return s.SetDiskEncryptionResetStatusFunc(ctx, hostID, status)
 }
 
-func (s *DataStore) UpdateHostMDMProfilesVerification(ctx context.Context, host *fleet.Host, verified []string, failed []string) error {
+func (s *DataStore) UpdateHostMDMProfilesVerification(ctx context.Context, hostUUID string, toVerify []string, toFail []string, toRetry []string) error {
 	s.mu.Lock()
 	s.UpdateHostMDMProfilesVerificationFuncInvoked = true
 	s.mu.Unlock()
-	return s.UpdateHostMDMProfilesVerificationFunc(ctx, host, verified, failed)
+	return s.UpdateHostMDMProfilesVerificationFunc(ctx, hostUUID, toVerify, toFail, toRetry)
 }
 
 func (s *DataStore) GetHostMDMProfilesExpectedForVerification(ctx context.Context, host *fleet.Host) (map[string]*fleet.ExpectedMDMProfile, error) {
@@ -3321,6 +3331,20 @@ func (s *DataStore) GetHostMDMProfilesExpectedForVerification(ctx context.Contex
 	return s.GetHostMDMProfilesExpectedForVerificationFunc(ctx, host)
 }
 
+func (s *DataStore) GetHostMDMProfilesRetryCounts(ctx context.Context, hostUUID string) ([]fleet.HostMDMProfileRetryCount, error) {
+	s.mu.Lock()
+	s.GetHostMDMProfilesRetryCountsFuncInvoked = true
+	s.mu.Unlock()
+	return s.GetHostMDMProfilesRetryCountsFunc(ctx, hostUUID)
+}
+
+func (s *DataStore) GetHostMDMProfileRetryCountByCommandUUID(ctx context.Context, hostUUID string, cmdUUID string) (fleet.HostMDMProfileRetryCount, error) {
+	s.mu.Lock()
+	s.GetHostMDMProfileRetryCountByCommandUUIDFuncInvoked = true
+	s.mu.Unlock()
+	return s.GetHostMDMProfileRetryCountByCommandUUIDFunc(ctx, hostUUID, cmdUUID)
+}
+
 func (s *DataStore) SetOrUpdateHostOrbitInfo(ctx context.Context, hostID uint, version string) error {
 	s.mu.Lock()
 	s.SetOrUpdateHostOrbitInfoFuncInvoked = true

server/service/apple_mdm.go

@@ -2380,13 +2380,14 @@ func (svc *MDMAppleCheckinAndCommandService) CommandAndReportResults(r *mdm.Requ
 
 	switch requestType {
 	case "InstallProfile":
-		return nil, svc.ds.UpdateOrDeleteHostMDMAppleProfile(r.Context, &fleet.HostMDMAppleProfile{
-			CommandUUID:   res.CommandUUID,
-			HostUUID:      res.UDID,
-			Status:        mdmAppleDeliveryStatusFromCommandStatus(res.Status),
-			Detail:        apple_mdm.FmtErrorChain(res.ErrorChain),
-			OperationType: fleet.MDMAppleOperationTypeInstall,
-		})
+		return nil, apple_mdm.HandleHostMDMProfileInstallResult(
+			r.Context,
+			svc.ds,
+			res.UDID,
+			res.CommandUUID,
+			mdmAppleDeliveryStatusFromCommandStatus(res.Status),
+			apple_mdm.FmtErrorChain(res.ErrorChain),
+		)
 	case "RemoveProfile":
 		return nil, svc.ds.UpdateOrDeleteHostMDMAppleProfile(r.Context, &fleet.HostMDMAppleProfile{
 			CommandUUID:   res.CommandUUID,

server/service/apple_mdm_test.go

@@ -1116,17 +1116,17 @@ func TestMDMCheckout(t *testing.T) {
 }
 
 func TestMDMCommandAndReportResultsProfileHandling(t *testing.T) {
-	ds := new(mock.Store)
-	svc := MDMAppleCheckinAndCommandService{ds: ds}
 	ctx := context.Background()
 	hostUUID := "ABC-DEF-GHI"
 	commandUUID := "COMMAND-UUID"
+	profileIdentifier := "PROFILE-IDENTIFIER"
 
 	cases := []struct {
 		status      string
 		requestType string
 		errors      []mdm.ErrorChain
 		want        *fleet.HostMDMAppleProfile
+		prevRetries uint
 	}{
 		{
 			status:      "Acknowledged",
@@ -1154,6 +1154,20 @@ func TestMDMCommandAndReportResultsProfileHandling(t *testing.T) {
 			errors: []mdm.ErrorChain{
 				{ErrorCode: 123, ErrorDomain: "testDomain", USEnglishDescription: "testMessage"},
 			},
+			prevRetries: 0, // expect to retry
+			want: &fleet.HostMDMAppleProfile{
+				Status:        &fleet.MDMAppleDeliveryPending,
+				Detail:        "",
+				OperationType: fleet.MDMAppleOperationTypeInstall,
+			},
+		},
+		{
+			status:      "Error",
+			requestType: "InstallProfile",
+			errors: []mdm.ErrorChain{
+				{ErrorCode: 123, ErrorDomain: "testDomain", USEnglishDescription: "testMessage"},
+			},
+			prevRetries: 1, // expect to fail
 			want: &fleet.HostMDMAppleProfile{
 				Status:        &fleet.MDMAppleDeliveryFailed,
 				Detail:        "testDomain (123): testMessage\n",
@@ -1185,32 +1199,59 @@ func TestMDMCommandAndReportResultsProfileHandling(t *testing.T) {
 		},
 	}
 
-	for _, c := range cases {
-		ds.GetMDMAppleCommandRequestTypeFunc = func(ctx context.Context, targetCmd string) (string, error) {
-			require.Equal(t, commandUUID, targetCmd)
-			return c.requestType, nil
-		}
+	for i, c := range cases {
+		t.Run(fmt.Sprintf("%s%s-%d", c.requestType, c.status, i), func(t *testing.T) {
+			ds := new(mock.Store)
+			svc := MDMAppleCheckinAndCommandService{ds: ds}
+			ds.GetMDMAppleCommandRequestTypeFunc = func(ctx context.Context, targetCmd string) (string, error) {
+				require.Equal(t, commandUUID, targetCmd)
+				return c.requestType, nil
+			}
+			ds.UpdateOrDeleteHostMDMAppleProfileFunc = func(ctx context.Context, profile *fleet.HostMDMAppleProfile) error {
+				c.want.CommandUUID = commandUUID
+				c.want.HostUUID = hostUUID
+				require.Equal(t, c.want, profile)
+				return nil
+			}
+			ds.GetHostMDMProfileRetryCountByCommandUUIDFunc = func(ctx context.Context, hstUUID, cmdUUID string) (fleet.HostMDMProfileRetryCount, error) {
+				require.Equal(t, hostUUID, hstUUID)
+				require.Equal(t, commandUUID, cmdUUID)
+				return fleet.HostMDMProfileRetryCount{ProfileIdentifier: profileIdentifier, Retries: c.prevRetries}, nil
+			}
+			ds.UpdateHostMDMProfilesVerificationFunc = func(ctx context.Context, hostUUID string, toVerify, toFail, toRetry []string) error {
+				require.Equal(t, hostUUID, hostUUID)
+				require.Nil(t, toVerify)
+				require.Nil(t, toFail)
+				require.ElementsMatch(t, toRetry, []string{profileIdentifier})
+				return nil
+			}
 
-		ds.UpdateOrDeleteHostMDMAppleProfileFunc = func(ctx context.Context, profile *fleet.HostMDMAppleProfile) error {
-			c.want.CommandUUID = commandUUID
-			c.want.HostUUID = hostUUID
-			require.Equal(t, c.want, profile)
-			return nil
-		}
-
-		_, err := svc.CommandAndReportResults(
-			&mdm.Request{Context: ctx},
-			&mdm.CommandResults{
-				Enrollment:  mdm.Enrollment{UDID: hostUUID},
-				CommandUUID: commandUUID,
-				Status:      c.status,
-				RequestType: c.requestType,
-				ErrorChain:  c.errors,
-			},
-		)
-		require.NoError(t, err)
-		require.True(t, ds.GetMDMAppleCommandRequestTypeFuncInvoked)
-		require.True(t, ds.UpdateOrDeleteHostMDMAppleProfileFuncInvoked)
+			_, err := svc.CommandAndReportResults(
+				&mdm.Request{Context: ctx},
+				&mdm.CommandResults{
+					Enrollment:  mdm.Enrollment{UDID: hostUUID},
+					CommandUUID: commandUUID,
+					Status:      c.status,
+					RequestType: c.requestType,
+					ErrorChain:  c.errors,
+				},
+			)
+			require.NoError(t, err)
+			require.True(t, ds.GetMDMAppleCommandRequestTypeFuncInvoked)
+			var shouldCheckCount, shouldRetry, shouldUpdateOrDelete bool
+			if c.requestType == "InstallProfile" && c.status == "Error" {
+				shouldCheckCount = true
+			}
+			if shouldCheckCount && c.prevRetries == uint(0) {
+				shouldRetry = true
+			}
+			if c.requestType == "RemoveProfile" || (c.requestType == "InstallProfile" && !shouldRetry) {
+				shouldUpdateOrDelete = true
+			}
+			require.Equal(t, shouldCheckCount, ds.GetHostMDMProfileRetryCountByCommandUUIDFuncInvoked)
+			require.Equal(t, shouldRetry, ds.UpdateHostMDMProfilesVerificationFuncInvoked)
+			require.Equal(t, shouldUpdateOrDelete, ds.UpdateOrDeleteHostMDMAppleProfileFuncInvoked)
+		})
 	}
 }
 

server/service/integration_mdm_test.go

@@ -268,6 +268,17 @@ func (s *integrationMDMTestSuite) mockDEPResponse(handler http.Handler) {
 	})
 }
 
+func (s *integrationMDMTestSuite) awaitTriggerProfileSchedule(t *testing.T, additionalWait time.Duration) {
+	ch := make(chan struct{})
+	s.onProfileScheduleDone = func() {
+		close(ch)
+	}
+	_, err := s.profileSchedule.Trigger()
+	require.NoError(t, err)
+	<-ch
+	time.Sleep(additionalWait)
+}
+
 func (s *integrationMDMTestSuite) TestGetBootstrapToken() {
 	// see https://developer.apple.com/documentation/devicemanagement/get_bootstrap_token
 	t := s.T()
@@ -481,20 +492,12 @@ func (s *integrationMDMTestSuite) TestProfileManagement() {
 
 	err := s.ds.ApplyEnrollSecrets(ctx, nil, []*fleet.EnrollSecret{{Secret: t.Name()}})
 	require.NoError(t, err)
-	var globalFleetdProfile bytes.Buffer
-	params := mobileconfig.FleetdProfileOptions{
-		EnrollSecret: t.Name(),
-		ServerURL:    s.server.URL,
-		PayloadType:  mobileconfig.FleetdConfigPayloadIdentifier,
-	}
-	err = mobileconfig.FleetdProfileTemplate.Execute(&globalFleetdProfile, params)
-	require.NoError(t, err)
 
 	globalProfiles := [][]byte{
 		mobileconfigForTest("N1", "I1"),
 		mobileconfigForTest("N2", "I2"),
 	}
-	wantGlobalProfiles := append(globalProfiles, globalFleetdProfile.Bytes())
+	wantGlobalProfiles := append(globalProfiles, setupExpectedFleetdProfile(t, s.server.URL, t.Name(), nil))
 
 	// add global profiles
 	s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: globalProfiles}, http.StatusNoContent)
@@ -513,11 +516,7 @@ func (s *integrationMDMTestSuite) TestProfileManagement() {
 	teamProfiles := [][]byte{
 		mobileconfigForTest("N3", "I3"),
 	}
-	var teamFleetdProfile bytes.Buffer
-	params.EnrollSecret = "team1_enroll_sec"
-	err = mobileconfig.FleetdProfileTemplate.Execute(&teamFleetdProfile, params)
-	require.NoError(t, err)
-	wantTeamProfiles := append(teamProfiles, teamFleetdProfile.Bytes())
+	wantTeamProfiles := append(teamProfiles, setupExpectedFleetdProfile(t, s.server.URL, "team1_enroll_sec", &tm.ID))
 	// add profiles to the team
 	s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: teamProfiles}, http.StatusNoContent, "team_id", strconv.Itoa(int(tm.ID)))
 
@@ -545,71 +544,11 @@ func (s *integrationMDMTestSuite) TestProfileManagement() {
 
 	// Create a host and then enroll to MDM.
 	host, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t)
-
-	triggerSchedule := func() {
-		ch := make(chan struct{})
-		s.onProfileScheduleDone = func() {
-			close(ch)
-		}
-		_, err := s.profileSchedule.Trigger()
-		require.NoError(t, err)
-		<-ch
-	}
-
-	origPush := s.pushProvider.PushFunc
-	defer func() { s.pushProvider.PushFunc = origPush }()
-	s.pushProvider.PushFunc = func(pushes []*mdm.Push) (map[string]*push.Response, error) {
-		require.Len(t, pushes, 1)
-		require.Equal(t, pushes[0].PushMagic, "pushmagic"+mdmDevice.SerialNumber)
-		res := map[string]*push.Response{
-			pushes[0].Token.String(): {
-				Id:  uuid.New().String(),
-				Err: nil,
-			},
-		}
-		return res, nil
-	}
-
-	checkNextPayloads := func() ([][]byte, []string) {
-		var cmd *micromdm.CommandPayload
-		installs := [][]byte{}
-		removes := []string{}
-
-		for {
-			// on the first run, cmd will be nil and we need to
-			// ping the server via idle
-			if cmd == nil {
-				cmd, err = mdmDevice.Idle()
-			} else {
-				cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID)
-			}
-			require.NoError(t, err)
-
-			// if after idle or acknowledge cmd is still nil, it
-			// means there aren't any commands left to run
-			if cmd == nil {
-				break
-			}
-
-			switch cmd.Command.RequestType {
-			case "InstallProfile":
-				installs = append(installs, cmd.Command.InstallProfile.Payload)
-			case "RemoveProfile":
-				removes = append(removes, cmd.Command.RemoveProfile.Identifier)
-
-			}
-		}
-
-		return installs, removes
-	}
+	setupPusher(s, t, mdmDevice)
 
 	// trigger a profile sync
-	triggerSchedule()
-
-	time.Sleep(5 * time.Second)
-
-	installs, removes := checkNextPayloads()
-
+	s.awaitTriggerProfileSchedule(t, 5*time.Second)
+	installs, removes := checkNextPayloads(t, mdmDevice, false)
 	// verify that we received all profiles
 	require.ElementsMatch(t, wantGlobalProfiles, installs)
 	require.Empty(t, removes)
@@ -619,9 +558,8 @@ func (s *integrationMDMTestSuite) TestProfileManagement() {
 	require.NoError(t, err)
 
 	// trigger a profile sync
-	triggerSchedule()
-
-	installs, removes = checkNextPayloads()
+	s.awaitTriggerProfileSchedule(t, 0)
+	installs, removes = checkNextPayloads(t, mdmDevice, false)
 	// verify that we should install the team profile
 	require.ElementsMatch(t, wantTeamProfiles, installs)
 	// verify that we should delete both profiles
@@ -636,17 +574,16 @@ func (s *integrationMDMTestSuite) TestProfileManagement() {
 	s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: teamProfiles}, http.StatusNoContent, "team_id", strconv.Itoa(int(tm.ID)))
 
 	// trigger a profile sync
-	triggerSchedule()
-	installs, removes = checkNextPayloads()
+	s.awaitTriggerProfileSchedule(t, 0)
+	installs, removes = checkNextPayloads(t, mdmDevice, false)
 	// verify that we should install the team profiles
 	require.ElementsMatch(t, wantTeamProfiles, installs)
 	// verify that we should delete the old team profiles
 	require.ElementsMatch(t, []string{"I3"}, removes)
 
 	// with no changes
-	_, err = s.profileSchedule.Trigger()
-	require.NoError(t, err)
-	installs, removes = checkNextPayloads()
+	s.awaitTriggerProfileSchedule(t, 0)
+	installs, removes = checkNextPayloads(t, mdmDevice, false)
 	require.Empty(t, installs)
 	require.Empty(t, removes)
 
@@ -672,6 +609,357 @@ func (s *integrationMDMTestSuite) TestProfileManagement() {
 	require.Equal(t, uint(0), noTeamSummaryResp.Verified)
 }
 
+func (s *integrationMDMTestSuite) TestProfileRetries() {
+	t := s.T()
+	ctx := context.Background()
+
+	enrollSecret := "test-profile-retries-secret"
+	err := s.ds.ApplyEnrollSecrets(ctx, nil, []*fleet.EnrollSecret{{Secret: enrollSecret}})
+	require.NoError(t, err)
+
+	testProfiles := [][]byte{
+		mobileconfigForTest("N1", "I1"),
+		mobileconfigForTest("N2", "I2"),
+	}
+	initialExpectedProfiles := append(testProfiles, setupExpectedFleetdProfile(t, s.server.URL, enrollSecret, nil))
+
+	h, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t)
+	setupPusher(s, t, mdmDevice)
+
+	expectedProfileStatuses := map[string]fleet.MDMAppleDeliveryStatus{
+		"I1": fleet.MDMAppleDeliveryVerifying,
+		"I2": fleet.MDMAppleDeliveryVerifying,
+		mobileconfig.FleetdConfigPayloadIdentifier: fleet.MDMAppleDeliveryVerifying,
+	}
+	checkProfilesStatus := func(t *testing.T) {
+		storedProfs, err := s.ds.GetHostMDMProfiles(ctx, h.UUID)
+		require.NoError(t, err)
+		require.Len(t, storedProfs, len(expectedProfileStatuses))
+		for _, p := range storedProfs {
+			want, ok := expectedProfileStatuses[p.Identifier]
+			require.True(t, ok, "unexpected profile: %s", p.Identifier)
+			require.Equal(t, want, *p.Status, "expected status %s but got %s for profile: %s", want, *p.Status, p.Identifier)
+		}
+	}
+
+	expectedRetryCounts := map[string]uint{
+		"I1": 0,
+		"I2": 0,
+		mobileconfig.FleetdConfigPayloadIdentifier: 0,
+	}
+	checkRetryCounts := func(t *testing.T) {
+		counts, err := s.ds.GetHostMDMProfilesRetryCounts(ctx, h.UUID)
+		require.NoError(t, err)
+		require.Len(t, counts, len(expectedRetryCounts))
+		for _, c := range counts {
+			want, ok := expectedRetryCounts[c.ProfileIdentifier]
+			require.True(t, ok, "unexpected profile: %s", c.ProfileIdentifier)
+			require.Equal(t, want, c.Retries, "expected retry count %d but got %d for profile: %s", want, c.Retries, c.ProfileIdentifier)
+		}
+	}
+
+	hostProfsByIdent := map[string]*fleet.HostMacOSProfile{
+		"I1": {
+			Identifier:  "I1",
+			DisplayName: "N1",
+			InstallDate: time.Now(),
+		},
+		"I2": {
+			Identifier:  "I2",
+			DisplayName: "N2",
+			InstallDate: time.Now(),
+		},
+		mobileconfig.FleetdConfigPayloadIdentifier: {
+			Identifier:  mobileconfig.FleetdConfigPayloadIdentifier,
+			DisplayName: "Fleetd configuration",
+			InstallDate: time.Now(),
+		},
+	}
+	reportHostProfs := func(t *testing.T, identifiers ...string) {
+		report := make(map[string]*fleet.HostMacOSProfile, len(hostProfsByIdent))
+		for _, ident := range identifiers {
+			report[ident] = hostProfsByIdent[ident]
+		}
+		require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, s.ds, h, report))
+	}
+
+	setProfileUpdatedAt := func(t *testing.T, updatedAt time.Time, identifiers ...interface{}) {
+		bindVars := strings.TrimSuffix(strings.Repeat("?, ", len(identifiers)), ", ")
+		stmt := fmt.Sprintf("UPDATE mdm_apple_configuration_profiles SET updated_at = ? WHERE identifier IN(%s)", bindVars)
+		args := append([]interface{}{updatedAt}, identifiers...)
+		mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error {
+			_, err := tx.ExecContext(ctx, stmt, args...)
+			return err
+		})
+	}
+
+	t.Run("retry after verifying", func(t *testing.T) {
+		// upload test profiles then simulate expired grace period by setting updated_at timestamp of profiles back by 48 hours
+		s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: testProfiles}, http.StatusNoContent)
+		setProfileUpdatedAt(t, time.Now().Add(-48*time.Hour), "I1", "I2", mobileconfig.FleetdConfigPayloadIdentifier)
+
+		// trigger initial profile sync and confirm that we received all profiles
+		s.awaitTriggerProfileSchedule(t, 5*time.Second)
+		installs, removes := checkNextPayloads(t, mdmDevice, false)
+		require.ElementsMatch(t, initialExpectedProfiles, installs)
+		require.Empty(t, removes)
+
+		checkProfilesStatus(t) // all profiles verifying
+		checkRetryCounts(t)    // no retries yet
+
+		// report osquery results with I2 missing and confirm I2 marked as pending and other profiles are marked as verified
+		reportHostProfs(t, "I1", mobileconfig.FleetdConfigPayloadIdentifier)
+		expectedProfileStatuses["I2"] = fleet.MDMAppleDeliveryPending
+		expectedProfileStatuses["I1"] = fleet.MDMAppleDeliveryVerified
+		expectedProfileStatuses[mobileconfig.FleetdConfigPayloadIdentifier] = fleet.MDMAppleDeliveryVerified
+		checkProfilesStatus(t)
+		expectedRetryCounts["I2"] = 1
+		checkRetryCounts(t)
+
+		// trigger a profile sync and confirm that the install profile command for I2 was resent
+		s.awaitTriggerProfileSchedule(t, 0)
+		installs, removes = checkNextPayloads(t, mdmDevice, false)
+		require.ElementsMatch(t, [][]byte{initialExpectedProfiles[1]}, installs)
+		require.Empty(t, removes)
+
+		// report osquery results with I2 present and confirm that all profiles are verified
+		reportHostProfs(t, "I1", "I2", mobileconfig.FleetdConfigPayloadIdentifier)
+		expectedProfileStatuses["I2"] = fleet.MDMAppleDeliveryVerified
+		checkProfilesStatus(t)
+		checkRetryCounts(t) // unchanged
+
+		// trigger a profile sync and confirm that no profiles were sent
+		s.awaitTriggerProfileSchedule(t, 0)
+		installs, removes = checkNextPayloads(t, mdmDevice, false)
+		require.Empty(t, installs)
+		require.Empty(t, removes)
+	})
+
+	t.Run("retry after verification", func(t *testing.T) {
+		// report osquery results with I1 missing and confirm that the I1 marked as pending (initial retry)
+		reportHostProfs(t, "I2", mobileconfig.FleetdConfigPayloadIdentifier)
+		expectedProfileStatuses["I1"] = fleet.MDMAppleDeliveryPending
+		checkProfilesStatus(t)
+		expectedRetryCounts["I1"] = 1
+		checkRetryCounts(t)
+
+		// trigger a profile sync and confirm that the install profile command for I1 was resent
+		s.awaitTriggerProfileSchedule(t, 0)
+		installs, removes := checkNextPayloads(t, mdmDevice, false)
+		require.ElementsMatch(t, [][]byte{initialExpectedProfiles[0]}, installs)
+		require.Empty(t, removes)
+
+		// report osquery results with I1 missing again and confirm that the I1 marked as failed (max retries exceeded)
+		reportHostProfs(t, "I2", mobileconfig.FleetdConfigPayloadIdentifier)
+		expectedProfileStatuses["I1"] = fleet.MDMAppleDeliveryFailed
+		checkProfilesStatus(t)
+		checkRetryCounts(t) // unchanged
+
+		// trigger a profile sync and confirm that the install profile command for I1 was not resent
+		s.awaitTriggerProfileSchedule(t, 0)
+		installs, removes = checkNextPayloads(t, mdmDevice, false)
+		require.Empty(t, installs)
+		require.Empty(t, removes)
+	})
+
+	t.Run("retry after device error", func(t *testing.T) {
+		// add another profile and set the updated_at timestamp back by 48 hours
+		newProfile := mobileconfigForTest("N3", "I3")
+		testProfiles = append(testProfiles, newProfile)
+		s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: testProfiles}, http.StatusNoContent)
+		setProfileUpdatedAt(t, time.Now().Add(-48*time.Hour), "I1", "I2", mobileconfig.FleetdConfigPayloadIdentifier, "I3")
+
+		// trigger a profile sync and confirm that the install profile command for I3 was sent and
+		// simulate a device error
+		s.awaitTriggerProfileSchedule(t, 0)
+		installs, removes := checkNextPayloads(t, mdmDevice, true)
+		require.ElementsMatch(t, [][]byte{newProfile}, installs)
+		require.Empty(t, removes)
+		expectedProfileStatuses["I3"] = fleet.MDMAppleDeliveryPending
+		checkProfilesStatus(t)
+		expectedRetryCounts["I3"] = 1
+		checkRetryCounts(t)
+
+		// trigger a profile sync and confirm that the install profile command for I3 was sent and
+		// simulate a device ack
+		s.awaitTriggerProfileSchedule(t, 0)
+		installs, removes = checkNextPayloads(t, mdmDevice, false)
+		require.ElementsMatch(t, [][]byte{newProfile}, installs)
+		require.Empty(t, removes)
+		expectedProfileStatuses["I3"] = fleet.MDMAppleDeliveryVerifying
+		checkProfilesStatus(t)
+		checkRetryCounts(t) // unchanged
+
+		// report osquery results with I3 missing and confirm that the I3 marked as failed (max
+		// retries exceeded)
+		reportHostProfs(t, "I2", mobileconfig.FleetdConfigPayloadIdentifier)
+		expectedProfileStatuses["I3"] = fleet.MDMAppleDeliveryFailed
+		checkProfilesStatus(t)
+		checkRetryCounts(t) // unchanged
+
+		// trigger a profile sync and confirm that the install profile command for I3 was not resent
+		s.awaitTriggerProfileSchedule(t, 0)
+		installs, removes = checkNextPayloads(t, mdmDevice, false)
+		require.Empty(t, installs)
+		require.Empty(t, removes)
+	})
+
+	t.Run("repeated device error", func(t *testing.T) {
+		// add another profile and set the updated_at timestamp back by 48 hours
+		newProfile := mobileconfigForTest("N4", "I4")
+		testProfiles = append(testProfiles, newProfile)
+		s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: testProfiles}, http.StatusNoContent)
+		setProfileUpdatedAt(t, time.Now().Add(-48*time.Hour), "I1", "I2", mobileconfig.FleetdConfigPayloadIdentifier, "I3", "I4")
+
+		// trigger a profile sync and confirm that the install profile command for I3 was sent and
+		// simulate a device error
+		s.awaitTriggerProfileSchedule(t, 0)
+		installs, removes := checkNextPayloads(t, mdmDevice, true)
+		require.ElementsMatch(t, [][]byte{newProfile}, installs)
+		require.Empty(t, removes)
+		expectedProfileStatuses["I4"] = fleet.MDMAppleDeliveryPending
+		checkProfilesStatus(t)
+		expectedRetryCounts["I4"] = 1
+		checkRetryCounts(t)
+
+		// trigger a profile sync and confirm that the install profile command for I4 was sent and
+		// simulate a second device error
+		s.awaitTriggerProfileSchedule(t, 0)
+		installs, removes = checkNextPayloads(t, mdmDevice, true)
+		require.ElementsMatch(t, [][]byte{newProfile}, installs)
+		require.Empty(t, removes)
+		expectedProfileStatuses["I4"] = fleet.MDMAppleDeliveryFailed
+		checkProfilesStatus(t)
+		checkRetryCounts(t) // unchanged
+
+		// trigger a profile sync and confirm that the install profile command for I3 was not resent
+		s.awaitTriggerProfileSchedule(t, 0)
+		installs, removes = checkNextPayloads(t, mdmDevice, false)
+		require.Empty(t, installs)
+		require.Empty(t, removes)
+	})
+
+	t.Run("retry count does not reset", func(t *testing.T) {
+		// add another profile and set the updated_at timestamp back by 48 hours
+		newProfile := mobileconfigForTest("N5", "I5")
+		testProfiles = append(testProfiles, newProfile)
+		hostProfsByIdent["I5"] = &fleet.HostMacOSProfile{Identifier: "I5", DisplayName: "N5", InstallDate: time.Now()}
+		s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: testProfiles}, http.StatusNoContent)
+		setProfileUpdatedAt(t, time.Now().Add(-48*time.Hour), "I1", "I2", mobileconfig.FleetdConfigPayloadIdentifier, "I3", "I4", "I5")
+
+		// trigger a profile sync and confirm that the install profile command for I3 was sent and
+		// simulate a device error
+		s.awaitTriggerProfileSchedule(t, 0)
+		installs, removes := checkNextPayloads(t, mdmDevice, true)
+		require.ElementsMatch(t, [][]byte{newProfile}, installs)
+		require.Empty(t, removes)
+		expectedProfileStatuses["I5"] = fleet.MDMAppleDeliveryPending
+		checkProfilesStatus(t)
+		expectedRetryCounts["I5"] = 1
+		checkRetryCounts(t)
+
+		// trigger a profile sync and confirm that the install profile command for I5 was sent and
+		// simulate a device ack
+		s.awaitTriggerProfileSchedule(t, 0)
+		installs, removes = checkNextPayloads(t, mdmDevice, false)
+		require.ElementsMatch(t, [][]byte{newProfile}, installs)
+		require.Empty(t, removes)
+		expectedProfileStatuses["I5"] = fleet.MDMAppleDeliveryVerifying
+		checkProfilesStatus(t)
+		checkRetryCounts(t) // unchanged
+
+		// report osquery results with I5 found and confirm that the I5 marked as verified
+		reportHostProfs(t, "I2", mobileconfig.FleetdConfigPayloadIdentifier, "I5")
+		expectedProfileStatuses["I5"] = fleet.MDMAppleDeliveryVerified
+		checkProfilesStatus(t)
+		checkRetryCounts(t) // unchanged
+
+		// trigger a profile sync and confirm that the install profile command for I5 was not resent
+		s.awaitTriggerProfileSchedule(t, 0)
+		installs, removes = checkNextPayloads(t, mdmDevice, false)
+		require.Empty(t, installs)
+		require.Empty(t, removes)
+
+		// report osquery results again, this time I5 is missing and confirm that the I5 marked as
+		// failed (max retries exceeded)
+		reportHostProfs(t, "I2", mobileconfig.FleetdConfigPayloadIdentifier)
+		expectedProfileStatuses["I5"] = fleet.MDMAppleDeliveryFailed
+		checkProfilesStatus(t)
+		checkRetryCounts(t) // unchanged
+
+		// trigger a profile sync and confirm that the install profile command for I5 was not resent
+		s.awaitTriggerProfileSchedule(t, 0)
+		installs, removes = checkNextPayloads(t, mdmDevice, false)
+		require.Empty(t, installs)
+		require.Empty(t, removes)
+	})
+}
+
+func checkNextPayloads(t *testing.T, mdmDevice *mdmtest.TestMDMClient, forceDeviceErr bool) ([][]byte, []string) {
+	var cmd *micromdm.CommandPayload
+	var err error
+	installs := [][]byte{}
+	removes := []string{}
+
+	// on the first run, cmd will be nil and we need to
+	// ping the server via idle
+	// if after idle or acknowledge cmd is still nil, it
+	// means there aren't any commands left to run
+	for {
+		if cmd == nil {
+			cmd, err = mdmDevice.Idle()
+		} else {
+			if forceDeviceErr {
+				cmd, err = mdmDevice.Err(cmd.CommandUUID, []mdm.ErrorChain{})
+			} else {
+				cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID)
+			}
+		}
+		require.NoError(t, err)
+
+		if cmd == nil {
+			break
+		}
+
+		switch cmd.Command.RequestType {
+		case "InstallProfile":
+			installs = append(installs, cmd.Command.InstallProfile.Payload)
+		case "RemoveProfile":
+			removes = append(removes, cmd.Command.RemoveProfile.Identifier)
+
+		}
+	}
+	return installs, removes
+}
+
+func setupExpectedFleetdProfile(t *testing.T, serverURL string, enrollSecret string, teamID *uint) []byte {
+	var b bytes.Buffer
+	params := mobileconfig.FleetdProfileOptions{
+		EnrollSecret: enrollSecret,
+		ServerURL:    serverURL,
+		PayloadType:  mobileconfig.FleetdConfigPayloadIdentifier,
+	}
+	err := mobileconfig.FleetdProfileTemplate.Execute(&b, params)
+	require.NoError(t, err)
+	return b.Bytes()
+}
+
+func setupPusher(s *integrationMDMTestSuite, t *testing.T, mdmDevice *mdmtest.TestMDMClient) {
+	origPush := s.pushProvider.PushFunc
+	s.pushProvider.PushFunc = func(pushes []*mdm.Push) (map[string]*push.Response, error) {
+		require.Len(t, pushes, 1)
+		require.Equal(t, pushes[0].PushMagic, "pushmagic"+mdmDevice.SerialNumber)
+		res := map[string]*push.Response{
+			pushes[0].Token.String(): {
+				Id:  uuid.New().String(),
+				Err: nil,
+			},
+		}
+		return res, nil
+	}
+	t.Cleanup(func() { s.pushProvider.PushFunc = origPush })
+}
+
 func (s *integrationMDMTestSuite) TestPuppetMatchPreassignProfiles() {
 	ctx := context.Background()
 	t := s.T()

server/service/osquery_utils/queries_test.go

@@ -1230,12 +1230,13 @@ func TestDirectIngestHostMacOSProfiles(t *testing.T) {
 		}
 		return expected, nil
 	}
-	ds.UpdateHostMDMProfilesVerificationFunc = func(ctx context.Context, host *fleet.Host, verified, failed []string) error {
-		require.Equal(t, h.ID, host.ID)
-		require.Equal(t, len(installedProfiles), len(verified))
-		require.Len(t, failed, 0)
+	ds.UpdateHostMDMProfilesVerificationFunc = func(ctx context.Context, hostUUID string, toVerify, toFailed, toRetry []string) error {
+		require.Equal(t, h.UUID, hostUUID)
+		require.Equal(t, len(installedProfiles), len(toVerify))
+		require.Len(t, toFailed, 0)
+		require.Len(t, toRetry, 0)
 		for _, p := range installedProfiles {
-			require.Contains(t, verified, p.Identifier)
+			require.Contains(t, toVerify, p.Identifier)
 		}
 		return nil
 	}