From 5935c0bb48fbdf3a05e0e6884b6175a032872c74 Mon Sep 17 00:00:00 2001 From: gillespi314 <73313222+gillespi314@users.noreply.github.com> Date: Tue, 12 Sep 2023 09:59:47 -0500 Subject: [PATCH] Add retries to MDM profile verification (#13811) --- changes/11099-mdm-profiles-retries | 2 + server/datastore/mysql/apple_mdm.go | 90 +++- server/datastore/mysql/apple_mdm_test.go | 160 ++++-- ...dRetriesColumnHostMdmAppleProfilesTable.go | 25 + ...iesColumnHostMdmAppleProfilesTable_test.go | 109 +++++ server/datastore/mysql/schema.sql | 5 +- server/fleet/datastore.go | 13 +- server/fleet/mdm.go | 7 + server/mdm/apple/profile_verifier.go | 112 ++++- server/mock/datastore_mock.go | 30 +- server/service/apple_mdm.go | 15 +- server/service/apple_mdm_test.go | 95 ++-- server/service/integration_mdm_test.go | 458 ++++++++++++++---- server/service/osquery_utils/queries_test.go | 11 +- 14 files changed, 946 insertions(+), 186 deletions(-) create mode 100644 changes/11099-mdm-profiles-retries create mode 100644 server/datastore/mysql/migrations/tables/20230911163618_AddRetriesColumnHostMdmAppleProfilesTable.go create mode 100644 server/datastore/mysql/migrations/tables/20230911163618_AddRetriesColumnHostMdmAppleProfilesTable_test.go diff --git a/changes/11099-mdm-profiles-retries b/changes/11099-mdm-profiles-retries new file mode 100644 index 0000000000..e4e6c65bb8 --- /dev/null +++ b/changes/11099-mdm-profiles-retries @@ -0,0 +1,2 @@ + - Updated MDM profile verification so that an install profile command will be retried once if the command + resulted in an error or if osquery cannot confirm that the expected profile is installed. \ No newline at end of file diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index 42d4b5f9fe..27a8f03b59 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -1593,22 +1593,59 @@ func (ds *Datastore) UpdateOrDeleteHostMDMAppleProfile(ctx context.Context, prof return err } -func (ds *Datastore) UpdateHostMDMProfilesVerification(ctx context.Context, host *fleet.Host, verified, failed []string) error { +func (ds *Datastore) UpdateHostMDMProfilesVerification(ctx context.Context, hostUUID string, toVerify, toFail, toRetry []string) error { return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error { - if err := setMDMProfilesVerifiedDB(ctx, tx, host, verified); err != nil { + if err := setMDMProfilesVerifiedDB(ctx, tx, hostUUID, toVerify); err != nil { return err } - if err := setMDMProfilesFailedDB(ctx, tx, host, failed); err != nil { + if err := setMDMProfilesFailedDB(ctx, tx, hostUUID, toFail); err != nil { + return err + } + if err := setMDMProfilesRetryDB(ctx, tx, hostUUID, toRetry); err != nil { return err } return nil }) } +// setMDMProfilesRetryDB sets the status of the given identifiers to retry (nil) and increments the retry count +func setMDMProfilesRetryDB(ctx context.Context, tx sqlx.ExtContext, hostUUID string, identifiers []string) error { + if len(identifiers) == 0 { + return nil + } + + stmt := ` +UPDATE + host_mdm_apple_profiles +SET + status = NULL, + detail = '', + retries = retries + 1 +WHERE + host_uuid = ? + AND operation_type = ? + AND profile_identifier IN(?)` + + args := []interface{}{ + hostUUID, + fleet.MDMAppleOperationTypeInstall, + identifiers, + } + stmt, args, err := sqlx.In(stmt, args...) + if err != nil { + return ctxerr.Wrap(ctx, err, "building sql statement to set retry host macOS profiles") + } + + if _, err := tx.ExecContext(ctx, stmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "setting retry host macOS profiles") + } + return nil +} + // setMDMProfilesFailedDB sets the status of the given identifiers to failed if the current status // is verifying or verified. It also sets the detail to a message indicating that the profile was // either verifying or verified. Only profiles with the install operation type are updated. -func setMDMProfilesFailedDB(ctx context.Context, tx sqlx.ExtContext, host *fleet.Host, identifiers []string) error { +func setMDMProfilesFailedDB(ctx context.Context, tx sqlx.ExtContext, hostUUID string, identifiers []string) error { if len(identifiers) == 0 { return nil } @@ -1630,7 +1667,7 @@ WHERE fleet.HostMDMProfileDetailFailedWasVerifying, fleet.HostMDMProfileDetailFailedWasVerified, fleet.MDMAppleDeliveryFailed, - host.UUID, + hostUUID, []interface{}{fleet.MDMAppleDeliveryVerifying, fleet.MDMAppleDeliveryVerified}, fleet.MDMAppleOperationTypeInstall, identifiers, @@ -1648,7 +1685,7 @@ WHERE // setMDMProfilesVerifiedDB sets the status of the given identifiers to verified if the current // status is verifying. Only profiles with the install operation type are updated. -func setMDMProfilesVerifiedDB(ctx context.Context, tx sqlx.ExtContext, host *fleet.Host, identifiers []string) error { +func setMDMProfilesVerifiedDB(ctx context.Context, tx sqlx.ExtContext, hostUUID string, identifiers []string) error { if len(identifiers) == 0 { return nil } @@ -1667,7 +1704,7 @@ WHERE args := []interface{}{ fleet.MDMAppleDeliveryVerified, - host.UUID, + hostUUID, []interface{}{fleet.MDMAppleDeliveryVerifying, fleet.MDMAppleDeliveryFailed}, fleet.MDMAppleOperationTypeInstall, identifiers, @@ -1720,6 +1757,45 @@ WHERE return byIdentifier, nil } +func (ds *Datastore) GetHostMDMProfilesRetryCounts(ctx context.Context, hostUUID string) ([]fleet.HostMDMProfileRetryCount, error) { + stmt := ` +SELECT + profile_identifier, + retries +FROM + host_mdm_apple_profiles hmap +WHERE + hmap.host_uuid = ?` + + var dest []fleet.HostMDMProfileRetryCount + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &dest, stmt, hostUUID); err != nil { + return nil, ctxerr.Wrap(ctx, err, fmt.Sprintf("getting retry counts for host %s", hostUUID)) + } + + return dest, nil +} + +func (ds *Datastore) GetHostMDMProfileRetryCountByCommandUUID(ctx context.Context, hostUUID, cmdUUID string) (fleet.HostMDMProfileRetryCount, error) { + stmt := ` +SELECT + profile_identifier, retries +FROM + host_mdm_apple_profiles hmap +WHERE + hmap.host_uuid = ? + AND hmap.command_uuid = ?` + + var dest fleet.HostMDMProfileRetryCount + if err := sqlx.GetContext(ctx, ds.reader(ctx), &dest, stmt, hostUUID, cmdUUID); err != nil { + if err == sql.ErrNoRows { + return dest, notFound("HostMDMCommand").WithMessage(fmt.Sprintf("command uuid %s not found for host uuid %s", cmdUUID, hostUUID)) + } + return dest, ctxerr.Wrap(ctx, err, fmt.Sprintf("getting retry count for host %s command uuid %s", hostUUID, cmdUUID)) + } + + return dest, nil +} + func subqueryHostsMacOSSettingsStatusFailing() (string, []interface{}) { sql := ` SELECT diff --git a/server/datastore/mysql/apple_mdm_test.go b/server/datastore/mysql/apple_mdm_test.go index 52e0b2fe28..69dd18717d 100644 --- a/server/datastore/mysql/apple_mdm_test.go +++ b/server/datastore/mysql/apple_mdm_test.go @@ -1449,6 +1449,10 @@ func upsertHostCPs( upserts := []*fleet.MDMAppleBulkUpsertHostProfilePayload{} for _, h := range hosts { for _, cp := range profiles { + csum := []byte("csum") + if cp.Checksum != nil { + csum = cp.Checksum + } payload := fleet.MDMAppleBulkUpsertHostProfilePayload{ ProfileID: cp.ProfileID, ProfileIdentifier: cp.Identifier, @@ -1457,7 +1461,7 @@ func upsertHostCPs( CommandUUID: "", OperationType: opType, Status: status, - Checksum: []byte("csum"), + Checksum: csum, } upserts = append(upserts, &payload) } @@ -3927,6 +3931,15 @@ func testSetVerifiedMacOSProfiles(t *testing.T, ds *Datastore) { } } + adHocSetVerifying := func(hostUUID, profileIndentifier string) { + ExecAdhocSQL(t, ds, func(tx sqlx.ExtContext) error { + _, err := tx.ExecContext(ctx, + `UPDATE host_mdm_apple_profiles SET status = ? WHERE host_uuid = ? AND profile_identifier = ?`, + fleet.MDMAppleDeliveryVerifying, hostUUID, profileIndentifier) + return err + }) + } + // initialize the host MDM profile statuses upsertHostCPs(hosts, []*fleet.MDMAppleConfigProfile{storedByIdentifier[cp1.Identifier]}, fleet.MDMAppleOperationTypeInstall, &fleet.MDMAppleDeliveryPending, ctx, ds, t) upsertHostCPs(hosts, []*fleet.MDMAppleConfigProfile{storedByIdentifier[cp2.Identifier]}, fleet.MDMAppleOperationTypeInstall, &fleet.MDMAppleDeliveryVerifying, ctx, ds, t) @@ -4021,7 +4034,7 @@ func testSetVerifiedMacOSProfiles(t *testing.T, ds *Datastore) { return err }) - // after the grace period, status changes to "failed" if a profile is missing (i.e. not installed) + // after the grace period and one retry attempt, status changes to "failed" if a profile is missing (i.e. not installed) require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, hosts[2], profilesByIdentifier([]*fleet.HostMacOSProfile{ { Identifier: cp1.Identifier, @@ -4034,10 +4047,27 @@ func testSetVerifiedMacOSProfiles(t *testing.T, ds *Datastore) { InstallDate: time.Now(), }, }))) - expectedHostMDMStatus[hosts[2].ID][cp3.Identifier] = fleet.MDMAppleDeliveryFailed // cp3 is missing + expectedHostMDMStatus[hosts[2].ID][cp3.Identifier] = fleet.MDMAppleDeliveryPending // first retry for cp3 + checkHostMDMProfileStatuses() + // simulate retry command acknowledged by setting status to "verifying" + adHocSetVerifying(hosts[2].UUID, cp3.Identifier) + // report osquery results again with cp3 still missing + require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, hosts[2], profilesByIdentifier([]*fleet.HostMacOSProfile{ + { + Identifier: cp1.Identifier, + DisplayName: cp1.Name, + InstallDate: time.Now(), + }, + { + Identifier: cp2.Identifier, + DisplayName: cp2.Name, + InstallDate: time.Now(), + }, + }))) + expectedHostMDMStatus[hosts[2].ID][cp3.Identifier] = fleet.MDMAppleDeliveryFailed // still missing after retry so expect cp3 to fail checkHostMDMProfileStatuses() - // after the grace period, status changes to "failed" if a profile is outdated (i.e. installed + // after the grace period and one retry attempt, status changes to "failed" if a profile is outdated (i.e. installed // before the updated at timestamp of the profile) require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, hosts[2], profilesByIdentifier([]*fleet.HostMacOSProfile{ { @@ -4051,7 +4081,24 @@ func testSetVerifiedMacOSProfiles(t *testing.T, ds *Datastore) { InstallDate: time.Now().Add(-48 * time.Hour), }, }))) - expectedHostMDMStatus[hosts[2].ID][cp2.Identifier] = fleet.MDMAppleDeliveryFailed // cp2 is outdated + expectedHostMDMStatus[hosts[2].ID][cp2.Identifier] = fleet.MDMAppleDeliveryPending // first retry for cp2 + checkHostMDMProfileStatuses() + // simulate retry command acknowledged by setting status to "verifying" + adHocSetVerifying(hosts[2].UUID, cp2.Identifier) + // report osquery results again with cp2 still outdated + require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, hosts[2], profilesByIdentifier([]*fleet.HostMacOSProfile{ + { + Identifier: cp1.Identifier, + DisplayName: cp1.Name, + InstallDate: time.Now(), + }, + { + Identifier: cp2.Identifier, + DisplayName: cp2.Name, + InstallDate: time.Now().Add(-48 * time.Hour), + }, + }))) + expectedHostMDMStatus[hosts[2].ID][cp2.Identifier] = fleet.MDMAppleDeliveryFailed // still outdated after retry so expect cp2 to fail checkHostMDMProfileStatuses() } @@ -4674,6 +4721,13 @@ func TestMDMProfileVerification(t *testing.T) { }) } + setRetries := func(t *testing.T, hostUUID string, retries uint) { + ExecAdhocSQL(t, ds, func(tx sqlx.ExtContext) error { + _, err := tx.ExecContext(ctx, `UPDATE host_mdm_apple_profiles SET retries = ? WHERE host_uuid = ?`, retries, hostUUID) + return err + }) + } + checkHostStatus := func(t *testing.T, h *fleet.Host, expectedStatus fleet.MDMAppleDeliveryStatus, expectedDetail string) error { gotProfs, err := ds.GetHostMDMProfiles(ctx, h.UUID) if err != nil { @@ -4694,32 +4748,43 @@ func TestMDMProfileVerification(t *testing.T) { return nil } - t.Run("MissingProfile", func(t *testing.T) { - // missing profile, verifying and verified statuses should change to failed after the grace period + initializeProfile := func(t *testing.T, h *fleet.Host, cp *fleet.MDMAppleConfigProfile, status fleet.MDMAppleDeliveryStatus, prevRetries uint) { + upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &status, ctx, ds, t) + require.NoError(t, checkHostStatus(t, h, status, "")) + setRetries(t, h.UUID, prevRetries) + } + + cleanupProfiles := func(t *testing.T) { + ExecAdhocSQL(t, ds, func(tx sqlx.ExtContext) error { + _, err := tx.ExecContext(ctx, `DELETE FROM mdm_apple_configuration_profiles; DELETE FROM host_mdm_apple_profiles`) + return err + }) + } + + t.Run("MissingProfileWithRetry", func(t *testing.T) { + defer cleanupProfiles(t) + // missing profile, verifying and verified statuses should change to failed after the grace + // period and one retry cases := []testCase{ { name: "PendingThenMissing", initialStatus: fleet.MDMAppleDeliveryPending, expectedStatus: fleet.MDMAppleDeliveryPending, // no change - expectedDetail: "", }, { name: "VerifyingThenMissing", initialStatus: fleet.MDMAppleDeliveryVerifying, expectedStatus: fleet.MDMAppleDeliveryFailed, // change to failed - expectedDetail: string(fleet.HostMDMProfileDetailFailedWasVerifying), }, { name: "VerifiedThenMissing", initialStatus: fleet.MDMAppleDeliveryVerified, expectedStatus: fleet.MDMAppleDeliveryFailed, // change to failed - expectedDetail: string(fleet.HostMDMProfileDetailFailedWasVerified), }, { name: "FailedThenMissing", initialStatus: fleet.MDMAppleDeliveryFailed, expectedStatus: fleet.MDMAppleDeliveryFailed, // no change - expectedDetail: "", }, } @@ -4730,8 +4795,7 @@ func TestMDMProfileVerification(t *testing.T) { var reportedProfiles []*fleet.HostMacOSProfile // no profiles reported for this test // initialize - upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t) - require.NoError(t, checkHostStatus(t, h, tc.initialStatus, "")) + initializeProfile(t, h, cp, tc.initialStatus, 0) // within grace period setProfileUpdatedAt(t, cp, twoMinutesAgo) @@ -4739,13 +4803,24 @@ func TestMDMProfileVerification(t *testing.T) { require.NoError(t, checkHostStatus(t, h, tc.initialStatus, "")) // if missing within grace period, no change // reinitialize - upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t) - require.NoError(t, checkHostStatus(t, h, tc.initialStatus, "")) + initializeProfile(t, h, cp, tc.initialStatus, 0) // outside grace period setProfileUpdatedAt(t, cp, twoHoursAgo) require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, h, profilesByIdentifier(reportedProfiles))) - require.NoError(t, checkHostStatus(t, h, tc.expectedStatus, tc.expectedDetail)) // grace period expired, check expected status + if tc.expectedStatus == fleet.MDMAppleDeliveryFailed { + // grace period expired, first failure gets retried so status should be pending and empty detail + require.NoError(t, checkHostStatus(t, h, fleet.MDMAppleDeliveryPending, ""), tc.name) + } + + if tc.initialStatus != fleet.MDMAppleDeliveryPending { + // after retry, assume successful install profile command so status should be verifying + upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &fleet.MDMAppleDeliveryVerifying, ctx, ds, t) + // report osquery results + require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, h, profilesByIdentifier(reportedProfiles))) + // now we see the expected status + require.NoError(t, checkHostStatus(t, h, tc.expectedStatus, string(fleet.HostMDMProfileDetailFailedWasVerifying)), tc.name) // grace period expired, max retries so check expected status + } } }) @@ -4782,6 +4857,8 @@ func TestMDMProfileVerification(t *testing.T) { for i, tc := range cases { t.Run(tc.name, func(t *testing.T) { + defer cleanupProfiles(t) + // setup h := test.NewHost(t, ds, tc.name, tc.name, tc.name, tc.name, twoMinutesAgo) cp := setupTestProfile(t, fmt.Sprintf("%s-%d", tc.name, i)) @@ -4793,18 +4870,16 @@ func TestMDMProfileVerification(t *testing.T) { }, } - // initialize - upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t) - require.NoError(t, checkHostStatus(t, h, tc.initialStatus, "")) + // initialize with no remaining retries + initializeProfile(t, h, cp, tc.initialStatus, 1) // within grace period setProfileUpdatedAt(t, cp, twoMinutesAgo) require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, h, profilesByIdentifier(reportedProfiles))) require.NoError(t, checkHostStatus(t, h, tc.initialStatus, "")) // outdated profiles are treated similar to missing profiles so status doesn't change if within grace period - // reinitalize - upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t) - require.NoError(t, checkHostStatus(t, h, tc.initialStatus, "")) + // reinitalize with no remaining retries + initializeProfile(t, h, cp, tc.initialStatus, 1) // outside grace period setProfileUpdatedAt(t, cp, twoHoursAgo) @@ -4845,6 +4920,8 @@ func TestMDMProfileVerification(t *testing.T) { for i, tc := range cases { t.Run(tc.name, func(t *testing.T) { + defer cleanupProfiles(t) + // setup h := test.NewHost(t, ds, tc.name, tc.name, tc.name, tc.name, twoMinutesAgo) cp := setupTestProfile(t, fmt.Sprintf("%s-%d", tc.name, i)) @@ -4856,18 +4933,16 @@ func TestMDMProfileVerification(t *testing.T) { }, } - // initialize - upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t) - require.NoError(t, checkHostStatus(t, h, tc.initialStatus, "")) + // initialize with no remaining retries + initializeProfile(t, h, cp, tc.initialStatus, 1) // within grace period setProfileUpdatedAt(t, cp, twoMinutesAgo) require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, h, profilesByIdentifier(reportedProfiles))) require.NoError(t, checkHostStatus(t, h, tc.expectedStatus, tc.expectedDetail)) // if found within grace period, verifying status can become verified so check expected status - // reinitialize - upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t) - require.NoError(t, checkHostStatus(t, h, tc.initialStatus, "")) + // reinitializewith no remaining retries + initializeProfile(t, h, cp, tc.initialStatus, 1) // outside grace period setProfileUpdatedAt(t, cp, twoHoursAgo) @@ -4908,6 +4983,8 @@ func TestMDMProfileVerification(t *testing.T) { for i, tc := range cases { t.Run(tc.name, func(t *testing.T) { + defer cleanupProfiles(t) + // setup h := test.NewHost(t, ds, tc.name, tc.name, tc.name, tc.name, twoMinutesAgo) cp := setupTestProfile(t, fmt.Sprintf("%s-%d", tc.name, i)) @@ -4924,18 +5001,16 @@ func TestMDMProfileVerification(t *testing.T) { }, } - // initialize - upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t) - require.NoError(t, checkHostStatus(t, h, tc.initialStatus, "")) + // initialize with no remaining retries + initializeProfile(t, h, cp, tc.initialStatus, 1) // within grace period setProfileUpdatedAt(t, cp, twoMinutesAgo) require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, h, profilesByIdentifier(reportedProfiles))) require.NoError(t, checkHostStatus(t, h, tc.expectedStatus, tc.expectedDetail)) // if found within grace period, verifying status can become verified so check expected status - // reinitialize - upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{cp}, fleet.MDMAppleOperationTypeInstall, &tc.initialStatus, ctx, ds, t) - require.NoError(t, checkHostStatus(t, h, tc.initialStatus, "")) + // reinitialize with no remaining retries + initializeProfile(t, h, cp, tc.initialStatus, 1) // outside grace period setProfileUpdatedAt(t, cp, twoHoursAgo) @@ -4946,6 +5021,8 @@ func TestMDMProfileVerification(t *testing.T) { }) t.Run("EarliestInstallDate", func(t *testing.T) { + defer cleanupProfiles(t) + hostString := "host-earliest-install-date" h := test.NewHost(t, ds, hostString, hostString, hostString, hostString, twoMinutesAgo) @@ -4967,27 +5044,24 @@ func TestMDMProfileVerification(t *testing.T) { } initialStatus := fleet.MDMAppleDeliveryVerifying - // initialize - upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{stored0}, fleet.MDMAppleOperationTypeInstall, &initialStatus, ctx, ds, t) - require.NoError(t, checkHostStatus(t, h, initialStatus, "")) + // initialize with no remaining retries + initializeProfile(t, h, stored0, initialStatus, 1) // within grace period setProfileUpdatedAt(t, stored0, twoMinutesAgo) // host is out of date but still within grace period require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, h, profilesByIdentifier(reportedProfiles))) require.NoError(t, checkHostStatus(t, h, fleet.MDMAppleDeliveryVerifying, "")) // no change - // reinitialize - upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{stored0}, fleet.MDMAppleOperationTypeInstall, &initialStatus, ctx, ds, t) - require.NoError(t, checkHostStatus(t, h, initialStatus, "")) + // reinitialize with no remaining retries + initializeProfile(t, h, stored0, initialStatus, 1) // outside grace period setProfileUpdatedAt(t, stored0, twoHoursAgo) // host is out of date and grace period has passed require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, ds, h, profilesByIdentifier(reportedProfiles))) require.NoError(t, checkHostStatus(t, h, fleet.MDMAppleDeliveryFailed, string(fleet.HostMDMProfileDetailFailedWasVerifying))) // set to failed - // reinitialize - upsertHostCPs([]*fleet.Host{h}, []*fleet.MDMAppleConfigProfile{stored0}, fleet.MDMAppleOperationTypeInstall, &initialStatus, ctx, ds, t) - require.NoError(t, checkHostStatus(t, h, initialStatus, "")) + // reinitialize with no remaining retries + initializeProfile(t, h, stored0, initialStatus, 1) // save a copy of the config profile to team 1 cp.TeamID = ptr.Uint(1) diff --git a/server/datastore/mysql/migrations/tables/20230911163618_AddRetriesColumnHostMdmAppleProfilesTable.go b/server/datastore/mysql/migrations/tables/20230911163618_AddRetriesColumnHostMdmAppleProfilesTable.go new file mode 100644 index 0000000000..b83d6288b3 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20230911163618_AddRetriesColumnHostMdmAppleProfilesTable.go @@ -0,0 +1,25 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20230911163618, Down_20230911163618) +} + +func Up_20230911163618(tx *sql.Tx) error { + stmt := ` +ALTER TABLE host_mdm_apple_profiles + ADD COLUMN retries TINYINT(3) UNSIGNED NOT NULL DEFAULT 0` + + if _, err := tx.Exec(stmt); err != nil { + return fmt.Errorf("add retries to host_mdm_apple_profiles: %w", err) + } + return nil +} + +func Down_20230911163618(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20230911163618_AddRetriesColumnHostMdmAppleProfilesTable_test.go b/server/datastore/mysql/migrations/tables/20230911163618_AddRetriesColumnHostMdmAppleProfilesTable_test.go new file mode 100644 index 0000000000..8743498a17 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20230911163618_AddRetriesColumnHostMdmAppleProfilesTable_test.go @@ -0,0 +1,109 @@ +package tables + +import ( + "bytes" + "testing" + + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" +) + +func TestUp_20230911163618(t *testing.T) { + db := applyUpToPrev(t) + insertStmt := ` +INSERT INTO host_mdm_apple_profiles ( + profile_id, + profile_identifier, + host_uuid, + status, + operation_type, + detail, + command_uuid, + profile_name, + checksum) +VALUES + (?, ?, ?, ?, ?, ?, ?, ?, ?)` + + args := []interface{}{ + 1, + "test-identifier", + "test-host-uuid", + fleet.MDMAppleDeliveryVerified, + fleet.MDMAppleOperationTypeInstall, + "test-detail", + "test-command-uuid", + "test-profile-name", + []byte("test-checksum"), + } + execNoErr(t, db, insertStmt, args...) + + applyNext(t, db) + + // retrieve the stored value + var hmap struct { + ProfileID uint `db:"profile_id"` + ProfileIdentifier string `db:"profile_identifier"` + HostUUID string `db:"host_uuid"` + Status *fleet.MDMAppleDeliveryStatus `db:"status"` + OperationType fleet.MDMAppleOperationType `db:"operation_type"` + Detail string `db:"detail"` + CommandUUID string `db:"command_uuid"` + ProfileName string `db:"profile_name"` + Checksum []byte `db:"checksum"` + Retries uint `db:"retries"` + } + + selectStmt := "SELECT * FROM host_mdm_apple_profiles WHERE host_uuid = ?" + require.NoError(t, db.Get(&hmap, selectStmt, "test-host-uuid")) + require.Equal(t, uint(1), hmap.ProfileID) + require.Equal(t, "test-identifier", hmap.ProfileIdentifier) + require.Equal(t, "test-host-uuid", hmap.HostUUID) + require.Equal(t, fleet.MDMAppleDeliveryVerified, *hmap.Status) + require.Equal(t, fleet.MDMAppleOperationTypeInstall, hmap.OperationType) + require.Equal(t, "test-detail", hmap.Detail) + require.Equal(t, "test-command-uuid", hmap.CommandUUID) + require.Equal(t, "test-profile-name", hmap.ProfileName) + require.True(t, bytes.HasPrefix(hmap.Checksum, []byte("test-checksum"))) + require.Equal(t, uint(0), hmap.Retries) + + insertStmt = ` +INSERT INTO host_mdm_apple_profiles ( + profile_id, + profile_identifier, + host_uuid, + status, + operation_type, + detail, + command_uuid, + profile_name, + checksum, + retries) +VALUES + (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + + args = []interface{}{ + 1, + "test-identifier", + "test-host-uuid-2", + fleet.MDMAppleDeliveryVerified, + fleet.MDMAppleOperationTypeInstall, + "test-detail", + "test-command-uuid-2", + "test-profile-name", + []byte("test-checksum"), + 1, + } + execNoErr(t, db, insertStmt, args...) + + require.NoError(t, db.Get(&hmap, selectStmt, "test-host-uuid-2")) + require.Equal(t, uint(1), hmap.ProfileID) + require.Equal(t, "test-identifier", hmap.ProfileIdentifier) + require.Equal(t, "test-host-uuid-2", hmap.HostUUID) + require.Equal(t, fleet.MDMAppleDeliveryVerified, *hmap.Status) + require.Equal(t, fleet.MDMAppleOperationTypeInstall, hmap.OperationType) + require.Equal(t, "test-detail", hmap.Detail) + require.Equal(t, "test-command-uuid-2", hmap.CommandUUID) + require.Equal(t, "test-profile-name", hmap.ProfileName) + require.True(t, bytes.HasPrefix(hmap.Checksum, []byte("test-checksum"))) + require.Equal(t, uint(1), hmap.Retries) +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 0ba00115ba..43c7ccd719 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -283,6 +283,7 @@ CREATE TABLE `host_mdm_apple_profiles` ( `command_uuid` varchar(127) COLLATE utf8mb4_unicode_ci NOT NULL, `profile_name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', `checksum` binary(16) NOT NULL, + `retries` tinyint(3) unsigned NOT NULL DEFAULT '0', PRIMARY KEY (`host_uuid`,`profile_id`), KEY `status` (`status`), KEY `operation_type` (`operation_type`), @@ -682,9 +683,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=205 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=206 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `mobile_device_management_solutions` ( diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 3d3a999a77..1a8a533d98 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -701,11 +701,20 @@ type Datastore interface { SetDiskEncryptionResetStatus(ctx context.Context, hostID uint, status bool) error - // UpdateVerificationHostMacOSProfiles updates status of macOS profiles installed on a given host to verified. - UpdateHostMDMProfilesVerification(ctx context.Context, host *Host, verified, failed []string) error + // UpdateVerificationHostMacOSProfiles updates status of macOS profiles installed on a given + // host. The toVerify, toFail, and toRetry slices contain the identifiers of the profiles that + // should be verified, failed, and retried, respectively. For each profile in the toRetry slice, + // the retries count is incremented by 1 and the status is set to null so that an install + // profile command is enqueued the next time the profile manager cron runs. + UpdateHostMDMProfilesVerification(ctx context.Context, hostUUID string, toVerify, toFail, toRetry []string) error // GetHostMDMProfilesExpected returns the expected MDM profiles for a given host. The map is // keyed by the profile identifier. GetHostMDMProfilesExpectedForVerification(ctx context.Context, host *Host) (map[string]*ExpectedMDMProfile, error) + // GetHostMDMProfilesRetryCounts returns a list of MDM profile retry counts for a given host. + GetHostMDMProfilesRetryCounts(ctx context.Context, hostUUID string) ([]HostMDMProfileRetryCount, error) + // GetHostMDMProfileRetryCountByCommandUUID returns the retry count for the specified + // host UUID and command UUID. + GetHostMDMProfileRetryCountByCommandUUID(ctx context.Context, hostUUID, cmdUUID string) (HostMDMProfileRetryCount, error) // SetOrUpdateHostOrbitInfo inserts of updates the orbit info for a host SetOrUpdateHostOrbitInfo(ctx context.Context, hostID uint, version string) error diff --git a/server/fleet/mdm.go b/server/fleet/mdm.go index 3d890cad6e..9bac698299 100644 --- a/server/fleet/mdm.go +++ b/server/fleet/mdm.go @@ -128,3 +128,10 @@ func (ep ExpectedMDMProfile) IsWithinGracePeriod(hostDetailUpdatedAt time.Time) gracePeriod := 1 * time.Hour return hostDetailUpdatedAt.Before(ep.EarliestInstallDate.Add(gracePeriod)) } + +// HostMDMProfileRetryCount represents the number of times Fleet has attempted to install +// the identified profile on a host. +type HostMDMProfileRetryCount struct { + ProfileIdentifier string `db:"profile_identifier"` + Retries uint `db:"retries"` +} diff --git a/server/mdm/apple/profile_verifier.go b/server/mdm/apple/profile_verifier.go index b5735e3dcc..40822984df 100644 --- a/server/mdm/apple/profile_verifier.go +++ b/server/mdm/apple/profile_verifier.go @@ -6,11 +6,61 @@ import ( "github.com/fleetdm/fleet/v4/server/fleet" ) +// Profile verification is a set of related processes that run on the Fleet server to ensure that +// the MDM profiles installed on a host are the ones expected by the Fleet server. Expected profiles +// comprise the profiles that belong to the host's assigned team (or no +// team, as applicable). +// +// The Fleet server enqueues commands to install profiles on hosts via the MDM +// protocol. The Fleet server periodically runs a cron that enqueues install profile +// commands for host profiles that do not have a verification status (i.e. status is null). +// Install profile commands may be enqueued as a result of a variety of events, such as when a host +// enrolls in Fleet, when a host's team membership changes, when a new profile is uploaded, when an +// existing profile is modified, or when a failed profile is retried. +// +// Verification status of a host profile can change in the following ways: +// +// - When an install profile command is enqueued by the server, the verification status is set to "pending". +// +// - When the results of an install profile command are reported via the MDM protocol, the Fleet server +// parses the results and updates the host's verification status for the applicable profile. If the +// command was acknowledged, the verification status is set to "verifying". If the command resulted +// in an error, the server determines if the profile should be retried (in which case, a new install profile +// command will be enqueued by the server) or marked as "failed" and updates the datastore accordingly. +// +// - When host details are reported via osquery, the Fleet server ingests a list of installed +// profiles and compares the reported profiles with the list of profiles expected to be +// installed on the host. Expected profiles comprise the profiles that belong to the host's assigned +// team (or no team, as applicable). If an expected profile is found, the verification status is +// set to "verified". If an expected profile is missing from the reported results, the server determines +// if the profile should be retried (in which case, a new install profile command will be enqueued by the server) +// or marked as "failed" and updates the datastore accordingly. + +// maxRetries is the maximum times an install profile command may be retried, after which marked as failed and no further +// attempts will be made to install the profile. +const maxRetries = 1 + // ProfileVerificationStore is the minimal interface required to get and update the verification // status of a host's MDM profiles. The Fleet Datastore satisfies this interface. type ProfileVerificationStore interface { + // GetHostMDMProfilesExpectedForVerification returns the expected MDM profiles for a given host. The map is + // keyed by the profile identifier. GetHostMDMProfilesExpectedForVerification(ctx context.Context, host *fleet.Host) (map[string]*fleet.ExpectedMDMProfile, error) - UpdateHostMDMProfilesVerification(ctx context.Context, host *fleet.Host, verified, failed []string) error + // GetHostMDMProfilesRetryCounts returns the retry counts for the specified host. + GetHostMDMProfilesRetryCounts(ctx context.Context, hostUUID string) ([]fleet.HostMDMProfileRetryCount, error) + // GetHostMDMProfileRetryCountByCommandUUID returns the retry count for the specified + // host UUID and command UUID. + GetHostMDMProfileRetryCountByCommandUUID(ctx context.Context, hostUUID, commandUUID string) (fleet.HostMDMProfileRetryCount, error) + // UpdateHostMDMProfilesVerification updates status of macOS profiles installed on a given + // host. The toVerify, toFail, and toRetry slices contain the identifiers of the profiles that + // should be verified, failed, and retried, respectively. For each profile in the toRetry slice, + // the retries count is incremented by 1 and the status is set to null so that an install + // profile command is enqueued the next time the profile manager cron runs. + UpdateHostMDMProfilesVerification(ctx context.Context, hostUUID string, toVerify, toFail, toRetry []string) error + // UpdateOrDeleteHostMDMAppleProfile updates information about a single + // profile status. It deletes the row if the profile operation is "remove" + // and the status is "verifying" (i.e. successfully removed). + UpdateOrDeleteHostMDMAppleProfile(ctx context.Context, profile *fleet.HostMDMAppleProfile) error } var _ ProfileVerificationStore = (fleet.Datastore)(nil) @@ -24,7 +74,7 @@ func VerifyHostMDMProfiles(ctx context.Context, ds ProfileVerificationStore, hos return err } - failed := make([]string, 0, len(expected)) + missing := make([]string, 0, len(expected)) verified := make([]string, 0, len(expected)) for key, ep := range expected { withinGracePeriod := ep.IsWithinGracePeriod(host.DetailUpdatedAt) @@ -32,19 +82,71 @@ func VerifyHostMDMProfiles(ctx context.Context, ds ProfileVerificationStore, hos if !ok { // expected profile is missing from host if !withinGracePeriod { - failed = append(failed, key) + missing = append(missing, key) } continue } if ip.InstallDate.Before(ep.EarliestInstallDate) { // installed profile is outdated if !withinGracePeriod { - failed = append(failed, key) + missing = append(missing, key) } continue } verified = append(verified, key) } - return ds.UpdateHostMDMProfilesVerification(ctx, host, verified, failed) + toFail := make([]string, 0, len(missing)) + toRetry := make([]string, 0, len(missing)) + if len(missing) > 0 { + counts, err := ds.GetHostMDMProfilesRetryCounts(ctx, host.UUID) + if err != nil { + return err + } + retriesByProfileIdentifier := make(map[string]uint, len(counts)) + for _, r := range counts { + retriesByProfileIdentifier[r.ProfileIdentifier] = r.Retries + } + for _, key := range missing { + if retriesByProfileIdentifier[key] < maxRetries { + // if we haven't hit the max retries, we set the host profile status to nil (which + // causes an install profile command to be enqueued the next time the profile + // manager cron runs) and increment the retry count + toRetry = append(toRetry, key) + } else { + // otherwise we set the host profile status to failed + toFail = append(toFail, key) + } + } + } + + return ds.UpdateHostMDMProfilesVerification(ctx, host.UUID, verified, toFail, toRetry) +} + +// HandleHostMDMProfileInstallResult ingests the result of an install profile command reported via +// the MDM protocol and updates the verification status in the datastore. It is intended to be +// called by the Fleet MDM checkin and command service install profile request handler. +func HandleHostMDMProfileInstallResult(ctx context.Context, ds ProfileVerificationStore, hostUUID string, cmdUUID string, status *fleet.MDMAppleDeliveryStatus, detail string) error { + if status != nil && *status == fleet.MDMAppleDeliveryFailed { + m, err := ds.GetHostMDMProfileRetryCountByCommandUUID(ctx, hostUUID, cmdUUID) + if err != nil { + return err + } + + if m.Retries < maxRetries { + // if we haven't hit the max retries, we set the host profile status to nil (which + // causes an install profile command to be enqueued the next time the profile + // manager cron runs) and increment the retry count + return ds.UpdateHostMDMProfilesVerification(ctx, hostUUID, nil, nil, []string{m.ProfileIdentifier}) + } + } + + // otherwise update status and detail as usual + return ds.UpdateOrDeleteHostMDMAppleProfile(ctx, &fleet.HostMDMAppleProfile{ + CommandUUID: cmdUUID, + HostUUID: hostUUID, + Status: status, + Detail: detail, + OperationType: fleet.MDMAppleOperationTypeInstall, + }) } diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index 90658dffdd..c3b29b222e 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -488,10 +488,14 @@ type GetHostDiskEncryptionKeyFunc func(ctx context.Context, hostID uint) (*fleet type SetDiskEncryptionResetStatusFunc func(ctx context.Context, hostID uint, status bool) error -type UpdateHostMDMProfilesVerificationFunc func(ctx context.Context, host *fleet.Host, verified []string, failed []string) error +type UpdateHostMDMProfilesVerificationFunc func(ctx context.Context, hostUUID string, toVerify []string, toFail []string, toRetry []string) error type GetHostMDMProfilesExpectedForVerificationFunc func(ctx context.Context, host *fleet.Host) (map[string]*fleet.ExpectedMDMProfile, error) +type GetHostMDMProfilesRetryCountsFunc func(ctx context.Context, hostUUID string) ([]fleet.HostMDMProfileRetryCount, error) + +type GetHostMDMProfileRetryCountByCommandUUIDFunc func(ctx context.Context, hostUUID string, cmdUUID string) (fleet.HostMDMProfileRetryCount, error) + type SetOrUpdateHostOrbitInfoFunc func(ctx context.Context, hostID uint, version string) error type ReplaceHostDeviceMappingFunc func(ctx context.Context, id uint, mappings []*fleet.HostDeviceMapping) error @@ -1386,6 +1390,12 @@ type DataStore struct { GetHostMDMProfilesExpectedForVerificationFunc GetHostMDMProfilesExpectedForVerificationFunc GetHostMDMProfilesExpectedForVerificationFuncInvoked bool + GetHostMDMProfilesRetryCountsFunc GetHostMDMProfilesRetryCountsFunc + GetHostMDMProfilesRetryCountsFuncInvoked bool + + GetHostMDMProfileRetryCountByCommandUUIDFunc GetHostMDMProfileRetryCountByCommandUUIDFunc + GetHostMDMProfileRetryCountByCommandUUIDFuncInvoked bool + SetOrUpdateHostOrbitInfoFunc SetOrUpdateHostOrbitInfoFunc SetOrUpdateHostOrbitInfoFuncInvoked bool @@ -3307,11 +3317,11 @@ func (s *DataStore) SetDiskEncryptionResetStatus(ctx context.Context, hostID uin return s.SetDiskEncryptionResetStatusFunc(ctx, hostID, status) } -func (s *DataStore) UpdateHostMDMProfilesVerification(ctx context.Context, host *fleet.Host, verified []string, failed []string) error { +func (s *DataStore) UpdateHostMDMProfilesVerification(ctx context.Context, hostUUID string, toVerify []string, toFail []string, toRetry []string) error { s.mu.Lock() s.UpdateHostMDMProfilesVerificationFuncInvoked = true s.mu.Unlock() - return s.UpdateHostMDMProfilesVerificationFunc(ctx, host, verified, failed) + return s.UpdateHostMDMProfilesVerificationFunc(ctx, hostUUID, toVerify, toFail, toRetry) } func (s *DataStore) GetHostMDMProfilesExpectedForVerification(ctx context.Context, host *fleet.Host) (map[string]*fleet.ExpectedMDMProfile, error) { @@ -3321,6 +3331,20 @@ func (s *DataStore) GetHostMDMProfilesExpectedForVerification(ctx context.Contex return s.GetHostMDMProfilesExpectedForVerificationFunc(ctx, host) } +func (s *DataStore) GetHostMDMProfilesRetryCounts(ctx context.Context, hostUUID string) ([]fleet.HostMDMProfileRetryCount, error) { + s.mu.Lock() + s.GetHostMDMProfilesRetryCountsFuncInvoked = true + s.mu.Unlock() + return s.GetHostMDMProfilesRetryCountsFunc(ctx, hostUUID) +} + +func (s *DataStore) GetHostMDMProfileRetryCountByCommandUUID(ctx context.Context, hostUUID string, cmdUUID string) (fleet.HostMDMProfileRetryCount, error) { + s.mu.Lock() + s.GetHostMDMProfileRetryCountByCommandUUIDFuncInvoked = true + s.mu.Unlock() + return s.GetHostMDMProfileRetryCountByCommandUUIDFunc(ctx, hostUUID, cmdUUID) +} + func (s *DataStore) SetOrUpdateHostOrbitInfo(ctx context.Context, hostID uint, version string) error { s.mu.Lock() s.SetOrUpdateHostOrbitInfoFuncInvoked = true diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index bf583c0d93..8854e1e476 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -2380,13 +2380,14 @@ func (svc *MDMAppleCheckinAndCommandService) CommandAndReportResults(r *mdm.Requ switch requestType { case "InstallProfile": - return nil, svc.ds.UpdateOrDeleteHostMDMAppleProfile(r.Context, &fleet.HostMDMAppleProfile{ - CommandUUID: res.CommandUUID, - HostUUID: res.UDID, - Status: mdmAppleDeliveryStatusFromCommandStatus(res.Status), - Detail: apple_mdm.FmtErrorChain(res.ErrorChain), - OperationType: fleet.MDMAppleOperationTypeInstall, - }) + return nil, apple_mdm.HandleHostMDMProfileInstallResult( + r.Context, + svc.ds, + res.UDID, + res.CommandUUID, + mdmAppleDeliveryStatusFromCommandStatus(res.Status), + apple_mdm.FmtErrorChain(res.ErrorChain), + ) case "RemoveProfile": return nil, svc.ds.UpdateOrDeleteHostMDMAppleProfile(r.Context, &fleet.HostMDMAppleProfile{ CommandUUID: res.CommandUUID, diff --git a/server/service/apple_mdm_test.go b/server/service/apple_mdm_test.go index c2d4450174..b6b3d68253 100644 --- a/server/service/apple_mdm_test.go +++ b/server/service/apple_mdm_test.go @@ -1116,17 +1116,17 @@ func TestMDMCheckout(t *testing.T) { } func TestMDMCommandAndReportResultsProfileHandling(t *testing.T) { - ds := new(mock.Store) - svc := MDMAppleCheckinAndCommandService{ds: ds} ctx := context.Background() hostUUID := "ABC-DEF-GHI" commandUUID := "COMMAND-UUID" + profileIdentifier := "PROFILE-IDENTIFIER" cases := []struct { status string requestType string errors []mdm.ErrorChain want *fleet.HostMDMAppleProfile + prevRetries uint }{ { status: "Acknowledged", @@ -1154,6 +1154,20 @@ func TestMDMCommandAndReportResultsProfileHandling(t *testing.T) { errors: []mdm.ErrorChain{ {ErrorCode: 123, ErrorDomain: "testDomain", USEnglishDescription: "testMessage"}, }, + prevRetries: 0, // expect to retry + want: &fleet.HostMDMAppleProfile{ + Status: &fleet.MDMAppleDeliveryPending, + Detail: "", + OperationType: fleet.MDMAppleOperationTypeInstall, + }, + }, + { + status: "Error", + requestType: "InstallProfile", + errors: []mdm.ErrorChain{ + {ErrorCode: 123, ErrorDomain: "testDomain", USEnglishDescription: "testMessage"}, + }, + prevRetries: 1, // expect to fail want: &fleet.HostMDMAppleProfile{ Status: &fleet.MDMAppleDeliveryFailed, Detail: "testDomain (123): testMessage\n", @@ -1185,32 +1199,59 @@ func TestMDMCommandAndReportResultsProfileHandling(t *testing.T) { }, } - for _, c := range cases { - ds.GetMDMAppleCommandRequestTypeFunc = func(ctx context.Context, targetCmd string) (string, error) { - require.Equal(t, commandUUID, targetCmd) - return c.requestType, nil - } + for i, c := range cases { + t.Run(fmt.Sprintf("%s%s-%d", c.requestType, c.status, i), func(t *testing.T) { + ds := new(mock.Store) + svc := MDMAppleCheckinAndCommandService{ds: ds} + ds.GetMDMAppleCommandRequestTypeFunc = func(ctx context.Context, targetCmd string) (string, error) { + require.Equal(t, commandUUID, targetCmd) + return c.requestType, nil + } + ds.UpdateOrDeleteHostMDMAppleProfileFunc = func(ctx context.Context, profile *fleet.HostMDMAppleProfile) error { + c.want.CommandUUID = commandUUID + c.want.HostUUID = hostUUID + require.Equal(t, c.want, profile) + return nil + } + ds.GetHostMDMProfileRetryCountByCommandUUIDFunc = func(ctx context.Context, hstUUID, cmdUUID string) (fleet.HostMDMProfileRetryCount, error) { + require.Equal(t, hostUUID, hstUUID) + require.Equal(t, commandUUID, cmdUUID) + return fleet.HostMDMProfileRetryCount{ProfileIdentifier: profileIdentifier, Retries: c.prevRetries}, nil + } + ds.UpdateHostMDMProfilesVerificationFunc = func(ctx context.Context, hostUUID string, toVerify, toFail, toRetry []string) error { + require.Equal(t, hostUUID, hostUUID) + require.Nil(t, toVerify) + require.Nil(t, toFail) + require.ElementsMatch(t, toRetry, []string{profileIdentifier}) + return nil + } - ds.UpdateOrDeleteHostMDMAppleProfileFunc = func(ctx context.Context, profile *fleet.HostMDMAppleProfile) error { - c.want.CommandUUID = commandUUID - c.want.HostUUID = hostUUID - require.Equal(t, c.want, profile) - return nil - } - - _, err := svc.CommandAndReportResults( - &mdm.Request{Context: ctx}, - &mdm.CommandResults{ - Enrollment: mdm.Enrollment{UDID: hostUUID}, - CommandUUID: commandUUID, - Status: c.status, - RequestType: c.requestType, - ErrorChain: c.errors, - }, - ) - require.NoError(t, err) - require.True(t, ds.GetMDMAppleCommandRequestTypeFuncInvoked) - require.True(t, ds.UpdateOrDeleteHostMDMAppleProfileFuncInvoked) + _, err := svc.CommandAndReportResults( + &mdm.Request{Context: ctx}, + &mdm.CommandResults{ + Enrollment: mdm.Enrollment{UDID: hostUUID}, + CommandUUID: commandUUID, + Status: c.status, + RequestType: c.requestType, + ErrorChain: c.errors, + }, + ) + require.NoError(t, err) + require.True(t, ds.GetMDMAppleCommandRequestTypeFuncInvoked) + var shouldCheckCount, shouldRetry, shouldUpdateOrDelete bool + if c.requestType == "InstallProfile" && c.status == "Error" { + shouldCheckCount = true + } + if shouldCheckCount && c.prevRetries == uint(0) { + shouldRetry = true + } + if c.requestType == "RemoveProfile" || (c.requestType == "InstallProfile" && !shouldRetry) { + shouldUpdateOrDelete = true + } + require.Equal(t, shouldCheckCount, ds.GetHostMDMProfileRetryCountByCommandUUIDFuncInvoked) + require.Equal(t, shouldRetry, ds.UpdateHostMDMProfilesVerificationFuncInvoked) + require.Equal(t, shouldUpdateOrDelete, ds.UpdateOrDeleteHostMDMAppleProfileFuncInvoked) + }) } } diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 8bf69c0972..40e6b6b443 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -268,6 +268,17 @@ func (s *integrationMDMTestSuite) mockDEPResponse(handler http.Handler) { }) } +func (s *integrationMDMTestSuite) awaitTriggerProfileSchedule(t *testing.T, additionalWait time.Duration) { + ch := make(chan struct{}) + s.onProfileScheduleDone = func() { + close(ch) + } + _, err := s.profileSchedule.Trigger() + require.NoError(t, err) + <-ch + time.Sleep(additionalWait) +} + func (s *integrationMDMTestSuite) TestGetBootstrapToken() { // see https://developer.apple.com/documentation/devicemanagement/get_bootstrap_token t := s.T() @@ -481,20 +492,12 @@ func (s *integrationMDMTestSuite) TestProfileManagement() { err := s.ds.ApplyEnrollSecrets(ctx, nil, []*fleet.EnrollSecret{{Secret: t.Name()}}) require.NoError(t, err) - var globalFleetdProfile bytes.Buffer - params := mobileconfig.FleetdProfileOptions{ - EnrollSecret: t.Name(), - ServerURL: s.server.URL, - PayloadType: mobileconfig.FleetdConfigPayloadIdentifier, - } - err = mobileconfig.FleetdProfileTemplate.Execute(&globalFleetdProfile, params) - require.NoError(t, err) globalProfiles := [][]byte{ mobileconfigForTest("N1", "I1"), mobileconfigForTest("N2", "I2"), } - wantGlobalProfiles := append(globalProfiles, globalFleetdProfile.Bytes()) + wantGlobalProfiles := append(globalProfiles, setupExpectedFleetdProfile(t, s.server.URL, t.Name(), nil)) // add global profiles s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: globalProfiles}, http.StatusNoContent) @@ -513,11 +516,7 @@ func (s *integrationMDMTestSuite) TestProfileManagement() { teamProfiles := [][]byte{ mobileconfigForTest("N3", "I3"), } - var teamFleetdProfile bytes.Buffer - params.EnrollSecret = "team1_enroll_sec" - err = mobileconfig.FleetdProfileTemplate.Execute(&teamFleetdProfile, params) - require.NoError(t, err) - wantTeamProfiles := append(teamProfiles, teamFleetdProfile.Bytes()) + wantTeamProfiles := append(teamProfiles, setupExpectedFleetdProfile(t, s.server.URL, "team1_enroll_sec", &tm.ID)) // add profiles to the team s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: teamProfiles}, http.StatusNoContent, "team_id", strconv.Itoa(int(tm.ID))) @@ -545,71 +544,11 @@ func (s *integrationMDMTestSuite) TestProfileManagement() { // Create a host and then enroll to MDM. host, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) - - triggerSchedule := func() { - ch := make(chan struct{}) - s.onProfileScheduleDone = func() { - close(ch) - } - _, err := s.profileSchedule.Trigger() - require.NoError(t, err) - <-ch - } - - origPush := s.pushProvider.PushFunc - defer func() { s.pushProvider.PushFunc = origPush }() - s.pushProvider.PushFunc = func(pushes []*mdm.Push) (map[string]*push.Response, error) { - require.Len(t, pushes, 1) - require.Equal(t, pushes[0].PushMagic, "pushmagic"+mdmDevice.SerialNumber) - res := map[string]*push.Response{ - pushes[0].Token.String(): { - Id: uuid.New().String(), - Err: nil, - }, - } - return res, nil - } - - checkNextPayloads := func() ([][]byte, []string) { - var cmd *micromdm.CommandPayload - installs := [][]byte{} - removes := []string{} - - for { - // on the first run, cmd will be nil and we need to - // ping the server via idle - if cmd == nil { - cmd, err = mdmDevice.Idle() - } else { - cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID) - } - require.NoError(t, err) - - // if after idle or acknowledge cmd is still nil, it - // means there aren't any commands left to run - if cmd == nil { - break - } - - switch cmd.Command.RequestType { - case "InstallProfile": - installs = append(installs, cmd.Command.InstallProfile.Payload) - case "RemoveProfile": - removes = append(removes, cmd.Command.RemoveProfile.Identifier) - - } - } - - return installs, removes - } + setupPusher(s, t, mdmDevice) // trigger a profile sync - triggerSchedule() - - time.Sleep(5 * time.Second) - - installs, removes := checkNextPayloads() - + s.awaitTriggerProfileSchedule(t, 5*time.Second) + installs, removes := checkNextPayloads(t, mdmDevice, false) // verify that we received all profiles require.ElementsMatch(t, wantGlobalProfiles, installs) require.Empty(t, removes) @@ -619,9 +558,8 @@ func (s *integrationMDMTestSuite) TestProfileManagement() { require.NoError(t, err) // trigger a profile sync - triggerSchedule() - - installs, removes = checkNextPayloads() + s.awaitTriggerProfileSchedule(t, 0) + installs, removes = checkNextPayloads(t, mdmDevice, false) // verify that we should install the team profile require.ElementsMatch(t, wantTeamProfiles, installs) // verify that we should delete both profiles @@ -636,17 +574,16 @@ func (s *integrationMDMTestSuite) TestProfileManagement() { s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: teamProfiles}, http.StatusNoContent, "team_id", strconv.Itoa(int(tm.ID))) // trigger a profile sync - triggerSchedule() - installs, removes = checkNextPayloads() + s.awaitTriggerProfileSchedule(t, 0) + installs, removes = checkNextPayloads(t, mdmDevice, false) // verify that we should install the team profiles require.ElementsMatch(t, wantTeamProfiles, installs) // verify that we should delete the old team profiles require.ElementsMatch(t, []string{"I3"}, removes) // with no changes - _, err = s.profileSchedule.Trigger() - require.NoError(t, err) - installs, removes = checkNextPayloads() + s.awaitTriggerProfileSchedule(t, 0) + installs, removes = checkNextPayloads(t, mdmDevice, false) require.Empty(t, installs) require.Empty(t, removes) @@ -672,6 +609,357 @@ func (s *integrationMDMTestSuite) TestProfileManagement() { require.Equal(t, uint(0), noTeamSummaryResp.Verified) } +func (s *integrationMDMTestSuite) TestProfileRetries() { + t := s.T() + ctx := context.Background() + + enrollSecret := "test-profile-retries-secret" + err := s.ds.ApplyEnrollSecrets(ctx, nil, []*fleet.EnrollSecret{{Secret: enrollSecret}}) + require.NoError(t, err) + + testProfiles := [][]byte{ + mobileconfigForTest("N1", "I1"), + mobileconfigForTest("N2", "I2"), + } + initialExpectedProfiles := append(testProfiles, setupExpectedFleetdProfile(t, s.server.URL, enrollSecret, nil)) + + h, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) + setupPusher(s, t, mdmDevice) + + expectedProfileStatuses := map[string]fleet.MDMAppleDeliveryStatus{ + "I1": fleet.MDMAppleDeliveryVerifying, + "I2": fleet.MDMAppleDeliveryVerifying, + mobileconfig.FleetdConfigPayloadIdentifier: fleet.MDMAppleDeliveryVerifying, + } + checkProfilesStatus := func(t *testing.T) { + storedProfs, err := s.ds.GetHostMDMProfiles(ctx, h.UUID) + require.NoError(t, err) + require.Len(t, storedProfs, len(expectedProfileStatuses)) + for _, p := range storedProfs { + want, ok := expectedProfileStatuses[p.Identifier] + require.True(t, ok, "unexpected profile: %s", p.Identifier) + require.Equal(t, want, *p.Status, "expected status %s but got %s for profile: %s", want, *p.Status, p.Identifier) + } + } + + expectedRetryCounts := map[string]uint{ + "I1": 0, + "I2": 0, + mobileconfig.FleetdConfigPayloadIdentifier: 0, + } + checkRetryCounts := func(t *testing.T) { + counts, err := s.ds.GetHostMDMProfilesRetryCounts(ctx, h.UUID) + require.NoError(t, err) + require.Len(t, counts, len(expectedRetryCounts)) + for _, c := range counts { + want, ok := expectedRetryCounts[c.ProfileIdentifier] + require.True(t, ok, "unexpected profile: %s", c.ProfileIdentifier) + require.Equal(t, want, c.Retries, "expected retry count %d but got %d for profile: %s", want, c.Retries, c.ProfileIdentifier) + } + } + + hostProfsByIdent := map[string]*fleet.HostMacOSProfile{ + "I1": { + Identifier: "I1", + DisplayName: "N1", + InstallDate: time.Now(), + }, + "I2": { + Identifier: "I2", + DisplayName: "N2", + InstallDate: time.Now(), + }, + mobileconfig.FleetdConfigPayloadIdentifier: { + Identifier: mobileconfig.FleetdConfigPayloadIdentifier, + DisplayName: "Fleetd configuration", + InstallDate: time.Now(), + }, + } + reportHostProfs := func(t *testing.T, identifiers ...string) { + report := make(map[string]*fleet.HostMacOSProfile, len(hostProfsByIdent)) + for _, ident := range identifiers { + report[ident] = hostProfsByIdent[ident] + } + require.NoError(t, apple_mdm.VerifyHostMDMProfiles(ctx, s.ds, h, report)) + } + + setProfileUpdatedAt := func(t *testing.T, updatedAt time.Time, identifiers ...interface{}) { + bindVars := strings.TrimSuffix(strings.Repeat("?, ", len(identifiers)), ", ") + stmt := fmt.Sprintf("UPDATE mdm_apple_configuration_profiles SET updated_at = ? WHERE identifier IN(%s)", bindVars) + args := append([]interface{}{updatedAt}, identifiers...) + mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error { + _, err := tx.ExecContext(ctx, stmt, args...) + return err + }) + } + + t.Run("retry after verifying", func(t *testing.T) { + // upload test profiles then simulate expired grace period by setting updated_at timestamp of profiles back by 48 hours + s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: testProfiles}, http.StatusNoContent) + setProfileUpdatedAt(t, time.Now().Add(-48*time.Hour), "I1", "I2", mobileconfig.FleetdConfigPayloadIdentifier) + + // trigger initial profile sync and confirm that we received all profiles + s.awaitTriggerProfileSchedule(t, 5*time.Second) + installs, removes := checkNextPayloads(t, mdmDevice, false) + require.ElementsMatch(t, initialExpectedProfiles, installs) + require.Empty(t, removes) + + checkProfilesStatus(t) // all profiles verifying + checkRetryCounts(t) // no retries yet + + // report osquery results with I2 missing and confirm I2 marked as pending and other profiles are marked as verified + reportHostProfs(t, "I1", mobileconfig.FleetdConfigPayloadIdentifier) + expectedProfileStatuses["I2"] = fleet.MDMAppleDeliveryPending + expectedProfileStatuses["I1"] = fleet.MDMAppleDeliveryVerified + expectedProfileStatuses[mobileconfig.FleetdConfigPayloadIdentifier] = fleet.MDMAppleDeliveryVerified + checkProfilesStatus(t) + expectedRetryCounts["I2"] = 1 + checkRetryCounts(t) + + // trigger a profile sync and confirm that the install profile command for I2 was resent + s.awaitTriggerProfileSchedule(t, 0) + installs, removes = checkNextPayloads(t, mdmDevice, false) + require.ElementsMatch(t, [][]byte{initialExpectedProfiles[1]}, installs) + require.Empty(t, removes) + + // report osquery results with I2 present and confirm that all profiles are verified + reportHostProfs(t, "I1", "I2", mobileconfig.FleetdConfigPayloadIdentifier) + expectedProfileStatuses["I2"] = fleet.MDMAppleDeliveryVerified + checkProfilesStatus(t) + checkRetryCounts(t) // unchanged + + // trigger a profile sync and confirm that no profiles were sent + s.awaitTriggerProfileSchedule(t, 0) + installs, removes = checkNextPayloads(t, mdmDevice, false) + require.Empty(t, installs) + require.Empty(t, removes) + }) + + t.Run("retry after verification", func(t *testing.T) { + // report osquery results with I1 missing and confirm that the I1 marked as pending (initial retry) + reportHostProfs(t, "I2", mobileconfig.FleetdConfigPayloadIdentifier) + expectedProfileStatuses["I1"] = fleet.MDMAppleDeliveryPending + checkProfilesStatus(t) + expectedRetryCounts["I1"] = 1 + checkRetryCounts(t) + + // trigger a profile sync and confirm that the install profile command for I1 was resent + s.awaitTriggerProfileSchedule(t, 0) + installs, removes := checkNextPayloads(t, mdmDevice, false) + require.ElementsMatch(t, [][]byte{initialExpectedProfiles[0]}, installs) + require.Empty(t, removes) + + // report osquery results with I1 missing again and confirm that the I1 marked as failed (max retries exceeded) + reportHostProfs(t, "I2", mobileconfig.FleetdConfigPayloadIdentifier) + expectedProfileStatuses["I1"] = fleet.MDMAppleDeliveryFailed + checkProfilesStatus(t) + checkRetryCounts(t) // unchanged + + // trigger a profile sync and confirm that the install profile command for I1 was not resent + s.awaitTriggerProfileSchedule(t, 0) + installs, removes = checkNextPayloads(t, mdmDevice, false) + require.Empty(t, installs) + require.Empty(t, removes) + }) + + t.Run("retry after device error", func(t *testing.T) { + // add another profile and set the updated_at timestamp back by 48 hours + newProfile := mobileconfigForTest("N3", "I3") + testProfiles = append(testProfiles, newProfile) + s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: testProfiles}, http.StatusNoContent) + setProfileUpdatedAt(t, time.Now().Add(-48*time.Hour), "I1", "I2", mobileconfig.FleetdConfigPayloadIdentifier, "I3") + + // trigger a profile sync and confirm that the install profile command for I3 was sent and + // simulate a device error + s.awaitTriggerProfileSchedule(t, 0) + installs, removes := checkNextPayloads(t, mdmDevice, true) + require.ElementsMatch(t, [][]byte{newProfile}, installs) + require.Empty(t, removes) + expectedProfileStatuses["I3"] = fleet.MDMAppleDeliveryPending + checkProfilesStatus(t) + expectedRetryCounts["I3"] = 1 + checkRetryCounts(t) + + // trigger a profile sync and confirm that the install profile command for I3 was sent and + // simulate a device ack + s.awaitTriggerProfileSchedule(t, 0) + installs, removes = checkNextPayloads(t, mdmDevice, false) + require.ElementsMatch(t, [][]byte{newProfile}, installs) + require.Empty(t, removes) + expectedProfileStatuses["I3"] = fleet.MDMAppleDeliveryVerifying + checkProfilesStatus(t) + checkRetryCounts(t) // unchanged + + // report osquery results with I3 missing and confirm that the I3 marked as failed (max + // retries exceeded) + reportHostProfs(t, "I2", mobileconfig.FleetdConfigPayloadIdentifier) + expectedProfileStatuses["I3"] = fleet.MDMAppleDeliveryFailed + checkProfilesStatus(t) + checkRetryCounts(t) // unchanged + + // trigger a profile sync and confirm that the install profile command for I3 was not resent + s.awaitTriggerProfileSchedule(t, 0) + installs, removes = checkNextPayloads(t, mdmDevice, false) + require.Empty(t, installs) + require.Empty(t, removes) + }) + + t.Run("repeated device error", func(t *testing.T) { + // add another profile and set the updated_at timestamp back by 48 hours + newProfile := mobileconfigForTest("N4", "I4") + testProfiles = append(testProfiles, newProfile) + s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: testProfiles}, http.StatusNoContent) + setProfileUpdatedAt(t, time.Now().Add(-48*time.Hour), "I1", "I2", mobileconfig.FleetdConfigPayloadIdentifier, "I3", "I4") + + // trigger a profile sync and confirm that the install profile command for I3 was sent and + // simulate a device error + s.awaitTriggerProfileSchedule(t, 0) + installs, removes := checkNextPayloads(t, mdmDevice, true) + require.ElementsMatch(t, [][]byte{newProfile}, installs) + require.Empty(t, removes) + expectedProfileStatuses["I4"] = fleet.MDMAppleDeliveryPending + checkProfilesStatus(t) + expectedRetryCounts["I4"] = 1 + checkRetryCounts(t) + + // trigger a profile sync and confirm that the install profile command for I4 was sent and + // simulate a second device error + s.awaitTriggerProfileSchedule(t, 0) + installs, removes = checkNextPayloads(t, mdmDevice, true) + require.ElementsMatch(t, [][]byte{newProfile}, installs) + require.Empty(t, removes) + expectedProfileStatuses["I4"] = fleet.MDMAppleDeliveryFailed + checkProfilesStatus(t) + checkRetryCounts(t) // unchanged + + // trigger a profile sync and confirm that the install profile command for I3 was not resent + s.awaitTriggerProfileSchedule(t, 0) + installs, removes = checkNextPayloads(t, mdmDevice, false) + require.Empty(t, installs) + require.Empty(t, removes) + }) + + t.Run("retry count does not reset", func(t *testing.T) { + // add another profile and set the updated_at timestamp back by 48 hours + newProfile := mobileconfigForTest("N5", "I5") + testProfiles = append(testProfiles, newProfile) + hostProfsByIdent["I5"] = &fleet.HostMacOSProfile{Identifier: "I5", DisplayName: "N5", InstallDate: time.Now()} + s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: testProfiles}, http.StatusNoContent) + setProfileUpdatedAt(t, time.Now().Add(-48*time.Hour), "I1", "I2", mobileconfig.FleetdConfigPayloadIdentifier, "I3", "I4", "I5") + + // trigger a profile sync and confirm that the install profile command for I3 was sent and + // simulate a device error + s.awaitTriggerProfileSchedule(t, 0) + installs, removes := checkNextPayloads(t, mdmDevice, true) + require.ElementsMatch(t, [][]byte{newProfile}, installs) + require.Empty(t, removes) + expectedProfileStatuses["I5"] = fleet.MDMAppleDeliveryPending + checkProfilesStatus(t) + expectedRetryCounts["I5"] = 1 + checkRetryCounts(t) + + // trigger a profile sync and confirm that the install profile command for I5 was sent and + // simulate a device ack + s.awaitTriggerProfileSchedule(t, 0) + installs, removes = checkNextPayloads(t, mdmDevice, false) + require.ElementsMatch(t, [][]byte{newProfile}, installs) + require.Empty(t, removes) + expectedProfileStatuses["I5"] = fleet.MDMAppleDeliveryVerifying + checkProfilesStatus(t) + checkRetryCounts(t) // unchanged + + // report osquery results with I5 found and confirm that the I5 marked as verified + reportHostProfs(t, "I2", mobileconfig.FleetdConfigPayloadIdentifier, "I5") + expectedProfileStatuses["I5"] = fleet.MDMAppleDeliveryVerified + checkProfilesStatus(t) + checkRetryCounts(t) // unchanged + + // trigger a profile sync and confirm that the install profile command for I5 was not resent + s.awaitTriggerProfileSchedule(t, 0) + installs, removes = checkNextPayloads(t, mdmDevice, false) + require.Empty(t, installs) + require.Empty(t, removes) + + // report osquery results again, this time I5 is missing and confirm that the I5 marked as + // failed (max retries exceeded) + reportHostProfs(t, "I2", mobileconfig.FleetdConfigPayloadIdentifier) + expectedProfileStatuses["I5"] = fleet.MDMAppleDeliveryFailed + checkProfilesStatus(t) + checkRetryCounts(t) // unchanged + + // trigger a profile sync and confirm that the install profile command for I5 was not resent + s.awaitTriggerProfileSchedule(t, 0) + installs, removes = checkNextPayloads(t, mdmDevice, false) + require.Empty(t, installs) + require.Empty(t, removes) + }) +} + +func checkNextPayloads(t *testing.T, mdmDevice *mdmtest.TestMDMClient, forceDeviceErr bool) ([][]byte, []string) { + var cmd *micromdm.CommandPayload + var err error + installs := [][]byte{} + removes := []string{} + + // on the first run, cmd will be nil and we need to + // ping the server via idle + // if after idle or acknowledge cmd is still nil, it + // means there aren't any commands left to run + for { + if cmd == nil { + cmd, err = mdmDevice.Idle() + } else { + if forceDeviceErr { + cmd, err = mdmDevice.Err(cmd.CommandUUID, []mdm.ErrorChain{}) + } else { + cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID) + } + } + require.NoError(t, err) + + if cmd == nil { + break + } + + switch cmd.Command.RequestType { + case "InstallProfile": + installs = append(installs, cmd.Command.InstallProfile.Payload) + case "RemoveProfile": + removes = append(removes, cmd.Command.RemoveProfile.Identifier) + + } + } + return installs, removes +} + +func setupExpectedFleetdProfile(t *testing.T, serverURL string, enrollSecret string, teamID *uint) []byte { + var b bytes.Buffer + params := mobileconfig.FleetdProfileOptions{ + EnrollSecret: enrollSecret, + ServerURL: serverURL, + PayloadType: mobileconfig.FleetdConfigPayloadIdentifier, + } + err := mobileconfig.FleetdProfileTemplate.Execute(&b, params) + require.NoError(t, err) + return b.Bytes() +} + +func setupPusher(s *integrationMDMTestSuite, t *testing.T, mdmDevice *mdmtest.TestMDMClient) { + origPush := s.pushProvider.PushFunc + s.pushProvider.PushFunc = func(pushes []*mdm.Push) (map[string]*push.Response, error) { + require.Len(t, pushes, 1) + require.Equal(t, pushes[0].PushMagic, "pushmagic"+mdmDevice.SerialNumber) + res := map[string]*push.Response{ + pushes[0].Token.String(): { + Id: uuid.New().String(), + Err: nil, + }, + } + return res, nil + } + t.Cleanup(func() { s.pushProvider.PushFunc = origPush }) +} + func (s *integrationMDMTestSuite) TestPuppetMatchPreassignProfiles() { ctx := context.Background() t := s.T() diff --git a/server/service/osquery_utils/queries_test.go b/server/service/osquery_utils/queries_test.go index 5168203e07..7fe41da2cc 100644 --- a/server/service/osquery_utils/queries_test.go +++ b/server/service/osquery_utils/queries_test.go @@ -1230,12 +1230,13 @@ func TestDirectIngestHostMacOSProfiles(t *testing.T) { } return expected, nil } - ds.UpdateHostMDMProfilesVerificationFunc = func(ctx context.Context, host *fleet.Host, verified, failed []string) error { - require.Equal(t, h.ID, host.ID) - require.Equal(t, len(installedProfiles), len(verified)) - require.Len(t, failed, 0) + ds.UpdateHostMDMProfilesVerificationFunc = func(ctx context.Context, hostUUID string, toVerify, toFailed, toRetry []string) error { + require.Equal(t, h.UUID, hostUUID) + require.Equal(t, len(installedProfiles), len(toVerify)) + require.Len(t, toFailed, 0) + require.Len(t, toRetry, 0) for _, p := range installedProfiles { - require.Contains(t, verified, p.Identifier) + require.Contains(t, toVerify, p.Identifier) } return nil }