Reconcile stuck Android MDM commands via AMAPI operations.get (#50177)

**Related issue:** Resolves #46145

changes/46145-android-command-reconcile.md

@@ -0,0 +1 @@
+- Fixed Android hosts staying stuck on a pending Lock, Wipe, or Clear passcode when Google never delivered the command's result to Fleet. Fleet now checks the command's outcome directly with Google once a day and updates the host, so the command can be re-issued.

cmd/fleet/cron.go

@@ -2567,6 +2567,36 @@ func newAndroidMDMDeviceReconcilerSchedule(
 	return s, nil
 }
 
+// newAndroidMDMCommandReconcilerSchedule periodically polls AMAPI for the outcome of Android MDM
+// commands (Lock, Wipe, Clear passcode) that are still pending because their Pub/Sub COMMAND
+// notification never arrived, so hosts don't stay stuck in a pending state.
+func newAndroidMDMCommandReconcilerSchedule(
+	ctx context.Context,
+	instanceID string,
+	ds fleet.Datastore,
+	logger *slog.Logger,
+	licenseKey string,
+	newActivityFn fleet.NewActivityFunc,
+) (*schedule.Schedule, error) {
+	const (
+		name = string(fleet.CronMDMAndroidCommandReconciler)
+		// Daily is enough: a dropped notification is rare, and a day of reconciliation lag is invisible
+		// next to the indefinite wait an affected host has otherwise.
+		defaultInterval = 24 * time.Hour
+	)
+
+	logger = logger.With("cron", name)
+	s := schedule.New(
+		ctx, name, instanceID, defaultInterval, ds, ds,
+		schedule.WithLogger(logger),
+		schedule.WithJob("reconcile_android_commands", func(ctx context.Context) error {
+			return android_svc.ReconcileAndroidCommands(ctx, ds, logger, licenseKey, newActivityFn)
+		}),
+	)
+
+	return s, nil
+}
+
 func cronEnableAndroidAppReportsOnDefaultPolicy(
 	ctx context.Context,
 	instanceID string,

cmd/fleet/cron_registration.go

@@ -270,6 +270,18 @@ func registerMDMCrons(ctx context.Context, deps cronSchedulesDeps) {
 		)
 	})
 
+	// Register Android MDM Command Reconciler schedule (recovers commands whose Pub/Sub notification was lost)
+	deps.register("failed to register mdm_android_command_reconciler schedule", func() (fleet.CronSchedule, error) {
+		return newAndroidMDMCommandReconcilerSchedule(
+			ctx,
+			deps.instanceID,
+			deps.ds,
+			deps.logger,
+			deps.config.License.Key,
+			deps.svc.NewActivity,
+		)
+	})
+
 	deps.register("failed to register enable_android_app_reports_on_default_policy cron", func() (fleet.CronSchedule, error) {
 		return cronEnableAndroidAppReportsOnDefaultPolicy(ctx, deps.instanceID, deps.ds, deps.logger, deps.androidSvc)
 	})

server/datastore/mysql/android.go

@@ -1296,6 +1296,30 @@ func (ds *Datastore) UpdateMDMAndroidCommandStatus(ctx context.Context, commandU
 	return nil
 }
 
+// ListPendingMDMAndroidCommands returns pending commands created before createdBefore, oldest first, capped at limit
+// rows. The reconciler cron uses the age cutoff to skip commands that Pub/Sub is still likely to deliver, and the limit
+// to bound how many AMAPI calls a single run makes.
+func (ds *Datastore) ListPendingMDMAndroidCommands(ctx context.Context, createdBefore time.Time, limit int) ([]*android.MDMAndroidCommand, error) {
+	const stmt = `
+		SELECT
+			command_uuid, host_uuid, operation_name, command_type, status,
+			error_code, error_message, created_at, updated_at
+		FROM mdm_android_commands
+		WHERE status = ? AND created_at < ?
+		-- command_uuid breaks ties so rows with identical created_at keep a stable order between runs,
+		-- otherwise a full batch could return the same subset every time and starve the rest.
+		ORDER BY created_at, command_uuid
+		LIMIT ?
+	`
+	var cmds []*android.MDMAndroidCommand
+	if err := sqlx.SelectContext(ctx, ds.reader(ctx), &cmds, stmt,
+		string(android.MDMAndroidCommandStatusPending), createdBefore, limit,
+	); err != nil {
+		return nil, ctxerr.Wrap(ctx, err, "listing pending mdm android commands")
+	}
+	return cmds, nil
+}
+
 // androidApplicableProfilesQuery computes, per host, the set of applicable profiles based on team and label scoping. Label
 // semantics must match the in-code Apple/Windows evaluator in server/mdm/reconcile: a dynamic label created after the host's
 // last label scan (h.label_updated_at < lbl.created_at) has unknown membership and preserves the host's current profile state —

server/datastore/mysql/android_test.go

@@ -54,6 +54,7 @@ func TestAndroid(t *testing.T) {
 		{"GetHostMDMAndroidProfiles", testGetHostMDMAndroidProfiles},
 		{"GetAndroidPolicyRequestByUUID", testGetAndroidPolicyRequestByUUID},
 		{"MDMAndroidCommandCRUD", testMDMAndroidCommandCRUD},
+		{"ListPendingMDMAndroidCommands", testListPendingMDMAndroidCommands},
 		{"LockWipeHostViaAndroidMDM", testLockWipeHostViaAndroidMDM},
 		{"ListHostMDMAndroidProfilesPendingInstallWithVersion", testListHostMDMAndroidProfilesPendingInstallWithVersion},
 		{"BulkDeleteMDMAndroidHostProfiles", testBulkDeleteMDMAndroidHostProfiles},
@@ -2806,6 +2807,82 @@ func testMDMAndroidCommandCRUD(t *testing.T, ds *Datastore) {
 	})
 }
 
+func testListPendingMDMAndroidCommands(t *testing.T, ds *Datastore) {
+	ctx := t.Context()
+
+	// insertCommand creates a command row and backdates created_at so the age cutoff can be exercised
+	// without waiting. Returns the command_uuid.
+	insertCommand := func(t *testing.T, status string, age time.Duration) string {
+		cmdUUID := uuid.NewString()
+		require.NoError(t, ds.NewMDMAndroidCommand(ctx, &android.MDMAndroidCommand{
+			CommandUUID:   cmdUUID,
+			HostUUID:      "host-" + cmdUUID,
+			OperationName: "enterprises/E1/devices/D1/operations/" + cmdUUID,
+			CommandType:   string(android.MDMAndroidCommandTypeLock),
+			Status:        status,
+		}))
+		ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
+			_, err := q.ExecContext(ctx,
+				`UPDATE mdm_android_commands SET created_at = NOW(6) - INTERVAL ? SECOND WHERE command_uuid = ?`,
+				int(age.Seconds()), cmdUUID)
+			return err
+		})
+		return cmdUUID
+	}
+
+	uuidsOf := func(cmds []*android.MDMAndroidCommand) []string {
+		got := make([]string, 0, len(cmds))
+		for _, cmd := range cmds {
+			got = append(got, cmd.CommandUUID)
+		}
+		return got
+	}
+
+	oldest := insertCommand(t, string(android.MDMAndroidCommandStatusPending), 72*time.Hour)
+	middle := insertCommand(t, string(android.MDMAndroidCommandStatusPending), 48*time.Hour)
+	newest := insertCommand(t, string(android.MDMAndroidCommandStatusPending), 25*time.Hour)
+	tooRecent := insertCommand(t, string(android.MDMAndroidCommandStatusPending), time.Hour)
+	acknowledged := insertCommand(t, string(android.MDMAndroidCommandStatusAcknowledged), 48*time.Hour)
+	errored := insertCommand(t, string(android.MDMAndroidCommandStatusError), 48*time.Hour)
+
+	t.Run("returns only pending rows older than the cutoff, oldest first", func(t *testing.T) {
+		cmds, err := ds.ListPendingMDMAndroidCommands(ctx, time.Now().Add(-24*time.Hour), 100)
+		require.NoError(t, err)
+		require.Equal(t, []string{oldest, middle, newest}, uuidsOf(cmds))
+		require.NotContains(t, uuidsOf(cmds), tooRecent)
+		require.NotContains(t, uuidsOf(cmds), acknowledged)
+		require.NotContains(t, uuidsOf(cmds), errored)
+	})
+
+	t.Run("limit caps the batch to the oldest rows", func(t *testing.T) {
+		cmds, err := ds.ListPendingMDMAndroidCommands(ctx, time.Now().Add(-24*time.Hour), 2)
+		require.NoError(t, err)
+		require.Equal(t, []string{oldest, middle}, uuidsOf(cmds))
+	})
+
+	t.Run("returns all fields needed to reconcile", func(t *testing.T) {
+		cmds, err := ds.ListPendingMDMAndroidCommands(ctx, time.Now().Add(-24*time.Hour), 1)
+		require.NoError(t, err)
+		require.Len(t, cmds, 1)
+		assert.Equal(t, oldest, cmds[0].CommandUUID)
+		assert.Equal(t, "host-"+oldest, cmds[0].HostUUID)
+		assert.Equal(t, "enterprises/E1/devices/D1/operations/"+oldest, cmds[0].OperationName)
+		assert.Equal(t, string(android.MDMAndroidCommandTypeLock), cmds[0].CommandType)
+		assert.Equal(t, string(android.MDMAndroidCommandStatusPending), cmds[0].Status)
+		// created_at drives the not-found grace period in the reconciler, so it has to come back
+		// populated. Only assert it predates the cutoff -- an exact age would be at the mercy of clock
+		// skew between the app and the database.
+		assert.False(t, cmds[0].CreatedAt.IsZero())
+		assert.True(t, cmds[0].CreatedAt.Before(time.Now().Add(-24*time.Hour)))
+	})
+
+	t.Run("no matching rows returns an empty slice", func(t *testing.T) {
+		cmds, err := ds.ListPendingMDMAndroidCommands(ctx, time.Now().Add(-365*24*time.Hour), 100)
+		require.NoError(t, err)
+		require.Empty(t, cmds)
+	})
+}
+
 // newBareAndroidHostForTest inserts a minimal android-platform host row. Use this for tests
 // that exercise the host_mdm_actions layer and don't need a populated android_devices row
 // (use createAndroidHost + ds.NewAndroidHost for that).

server/datastore/mysql/migrations/tables/20260807151355_AddAndroidCommandsStatusCreatedAtIndex.go

@@ -0,0 +1,39 @@
+package tables
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20260807151355, Down_20260807151355)
+}
+
+// Up_20260805182836 adds an index supporting the Android command reconciler's batch query
+// (ListPendingMDMAndroidCommands), which reads
+//
+//	WHERE status = 'pending' AND created_at < ? ORDER BY created_at, command_uuid LIMIT ?
+//
+// mdm_android_commands only had the primary key, the operation_name unique key, and a host_uuid
+// key, none of which lead with status, so that query was a full table scan. The table grows with
+// every Lock/Wipe/Clear-passcode ever issued while the pending rows the cron wants are a small
+// slice of it, so the scan gets steadily more expensive as the table grows.
+//
+// status (equality) leads, created_at (range) follows -- the order MySQL needs to use both
+// predicates from one index. InnoDB appends the primary key (command_uuid) to every secondary
+// index, so this also satisfies the ORDER BY and the LIMIT can stop early instead of sorting.
+//
+// ALGORITHM=INPLACE, LOCK=NONE so the index builds without blocking command inserts.
+func Up_20260807151355(tx *sql.Tx) error {
+	stmt := `ALTER TABLE mdm_android_commands
+		ADD INDEX idx_mdm_android_commands_status_created_at (status, created_at),
+		ALGORITHM=INPLACE, LOCK=NONE`
+	if _, err := tx.Exec(stmt); err != nil {
+		return fmt.Errorf("failed to add idx_mdm_android_commands_status_created_at: %w", err)
+	}
+	return nil
+}
+
+func Down_20260807151355(tx *sql.Tx) error {
+	return nil
+}

server/datastore/mysql/migrations/tables/20260807151355_AddAndroidCommandsStatusCreatedAtIndex_test.go

@@ -0,0 +1,44 @@
+package tables
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUp_20260807151355(t *testing.T) {
+	db := applyUpToPrev(t)
+
+	// Seed a command so the migration is exercised against a non-empty table.
+	execNoErr(t, db, `
+		INSERT INTO mdm_android_commands (command_uuid, host_uuid, operation_name, command_type, status)
+		VALUES ('cmd-uuid-1', 'host-uuid-1', 'enterprises/e1/devices/d1/operations/op1', 'LOCK', 'pending')
+	`)
+
+	applyNext(t, db)
+
+	rows, err := db.Query(
+		`SELECT column_name FROM information_schema.statistics
+		 WHERE table_schema = DATABASE() AND table_name = 'mdm_android_commands'
+		   AND index_name = 'idx_mdm_android_commands_status_created_at'
+		 ORDER BY seq_in_index`,
+	)
+	require.NoError(t, err)
+	defer rows.Close()
+
+	var columns []string
+	for rows.Next() {
+		var columnName string
+		require.NoError(t, rows.Scan(&columnName))
+		columns = append(columns, columnName)
+	}
+	require.NoError(t, rows.Err())
+	require.Equal(t, []string{"status", "created_at"}, columns)
+
+	// The seeded row survives the ALTER and is still readable through the new index's predicate.
+	var count int
+	require.NoError(t, db.QueryRow(
+		`SELECT COUNT(*) FROM mdm_android_commands WHERE status = 'pending' AND created_at < NOW(6)`,
+	).Scan(&count))
+	require.Equal(t, 1, count)
+}

server/fleet/cron_schedules.go

@@ -73,6 +73,9 @@ const (
 	CronChartDataCollection                     CronScheduleName = "chart_data_collection" // Used by chart bounded context
 	CronCleanupExpiredADUEChallenges            CronScheduleName = "cleanup_expired_adue_challenges"
 	CronAppleMDMOSUpdatesSchedule               CronScheduleName = "apple_mdm_os_updates"
+	// CronMDMAndroidCommandReconciler polls AMAPI for the outcome of Android MDM commands whose Pub/Sub
+	// COMMAND notification never arrived, so they don't stay pending forever. Runs every 24h.
+	CronMDMAndroidCommandReconciler CronScheduleName = "mdm_android_command_reconciler"
 )
 
 type CronSchedulesService interface {

server/fleet/datastore.go

@@ -3964,6 +3964,11 @@ type AndroidDatastore interface {
 	// a previously-issued command. Called by the Pub/Sub COMMAND handler on ack/error.
 	UpdateMDMAndroidCommandStatus(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error
 
+	// ListPendingMDMAndroidCommands returns commands still in the pending status that were created
+	// before createdBefore, oldest first, capped at limit rows. Used by the command reconciler cron to
+	// find commands whose Pub/Sub COMMAND notification never arrived.
+	ListPendingMDMAndroidCommands(ctx context.Context, createdBefore time.Time, limit int) ([]*android.MDMAndroidCommand, error)
+
 	// LockHostViaAndroidMDM inserts the LOCK row into mdm_android_commands and writes the lock_ref on host_mdm_actions in a
 	// single transaction, mirroring WipeHostViaWindowsMDM. The caller must populate cmd.CommandUUID and cmd.OperationName
 	// (returned by EnterprisesDevicesIssueCommand) before invoking.

server/mdm/android/mock/client.go

@@ -27,6 +27,8 @@ type EnterprisesDevicesDeleteFunc func(ctx context.Context, deviceName string) e
 
 type EnterprisesDevicesIssueCommandFunc func(ctx context.Context, deviceName string, command *androidmanagement.Command) (*androidmanagement.Operation, error)
 
+type EnterprisesDevicesOperationsGetFunc func(ctx context.Context, operationName string) (*androidmanagement.Operation, error)
+
 type EnterprisesDevicesListPartialFunc func(ctx context.Context, enterpriseName string, pageToken string) (*androidmanagement.ListDevicesResponse, error)
 
 type EnterprisesEnrollmentTokensCreateFunc func(ctx context.Context, enterpriseName string, token *androidmanagement.EnrollmentToken) (*androidmanagement.EnrollmentToken, error)
@@ -67,6 +69,9 @@ type Client struct {
 	EnterprisesDevicesIssueCommandFunc        EnterprisesDevicesIssueCommandFunc
 	EnterprisesDevicesIssueCommandFuncInvoked bool
 
+	EnterprisesDevicesOperationsGetFunc        EnterprisesDevicesOperationsGetFunc
+	EnterprisesDevicesOperationsGetFuncInvoked bool
+
 	EnterprisesDevicesListPartialFunc        EnterprisesDevicesListPartialFunc
 	EnterprisesDevicesListPartialFuncInvoked bool
 
@@ -146,6 +151,13 @@ func (p *Client) EnterprisesDevicesIssueCommand(ctx context.Context, deviceName
 	return p.EnterprisesDevicesIssueCommandFunc(ctx, deviceName, command)
 }
 
+func (p *Client) EnterprisesDevicesOperationsGet(ctx context.Context, operationName string) (*androidmanagement.Operation, error) {
+	p.mu.Lock()
+	p.EnterprisesDevicesOperationsGetFuncInvoked = true
+	p.mu.Unlock()
+	return p.EnterprisesDevicesOperationsGetFunc(ctx, operationName)
+}
+
 func (p *Client) EnterprisesDevicesListPartial(ctx context.Context, enterpriseName string, pageToken string) (*androidmanagement.ListDevicesResponse, error) {
 	p.mu.Lock()
 	p.EnterprisesDevicesListPartialFuncInvoked = true

server/mdm/android/service/androidmgmt/client.go

@@ -46,6 +46,13 @@ type Client interface {
 	// https://developers.google.com/android/management/reference/rest/v1/enterprises.devices/issueCommand
 	EnterprisesDevicesIssueCommand(ctx context.Context, deviceName string, command *androidmanagement.Command) (*androidmanagement.Operation, error)
 
+	// EnterprisesDevicesOperationsGet fetches the current state of an Operation returned by
+	// EnterprisesDevicesIssueCommand. It is the authoritative source for a command's outcome and lets
+	// Fleet reconcile commands whose Pub/Sub COMMAND notification never arrived. operationName is the
+	// full AMAPI resource name (enterprises/X/devices/Y/operations/Z). See:
+	// https://developers.google.com/android/management/reference/rest/v1/enterprises.devices.operations/get
+	EnterprisesDevicesOperationsGet(ctx context.Context, operationName string) (*androidmanagement.Operation, error)
+
 	// EnterprisesDevicesListPartial lists devices for the given enterprise with partial fields.
 	// Page size of 100 devices
 	// See: https://developers.google.com/android/management/reference/rest/v1/enterprises.devices/list
@@ -116,9 +123,36 @@ func IsNotModifiedError(err error) bool {
 // IsBadRequestError reports whether the AMAPI error indicates that the
 // request was invalid due to a client error.
 func IsBadRequestError(err error) bool {
-	var ae *googleapi.Error
-	if errors.As(err, &ae) {
+	if ae, ok := errors.AsType[*googleapi.Error](err); ok {
 		return ae.Code == http.StatusBadRequest
 	}
 	return false
 }
+
+// IsNotFoundError reports whether the AMAPI error indicates that the requested
+// resource does not exist.
+func IsNotFoundError(err error) bool {
+	if ae, ok := errors.AsType[*googleapi.Error](err); ok {
+		return ae.Code == http.StatusNotFound
+	}
+	return false
+}
+
+// IsAuthenticationError reports whether the AMAPI error indicates that the
+// request was rejected over credentials or access, rather than anything about
+// the resource that was requested.
+func IsAuthenticationError(err error) bool {
+	if ae, ok := errors.AsType[*googleapi.Error](err); ok {
+		return ae.Code == http.StatusUnauthorized || ae.Code == http.StatusForbidden
+	}
+	return false
+}
+
+// IsTooManyRequestsError reports whether the AMAPI error indicates that we
+// exceeded the project's request quota.
+func IsTooManyRequestsError(err error) bool {
+	if ae, ok := errors.AsType[*googleapi.Error](err); ok {
+		return ae.Code == http.StatusTooManyRequests
+	}
+	return false
+}

server/mdm/android/service/androidmgmt/google_client.go

@@ -246,6 +246,15 @@ func (g *GoogleClient) EnterprisesDevicesIssueCommand(ctx context.Context, devic
 	return op, nil
 }
 
+func (g *GoogleClient) EnterprisesDevicesOperationsGet(ctx context.Context, operationName string) (*androidmanagement.Operation, error) {
+	op, err := g.mgmt.Enterprises.Devices.Operations.Get(operationName).Context(ctx).Do()
+	if err != nil {
+		// Wrapped with %w so callers can classify the googleapi.Error (not found, quota exceeded).
+		return nil, fmt.Errorf("getting operation %s: %w", operationName, err)
+	}
+	return op, nil
+}
+
 func (g *GoogleClient) EnterprisesDevicesListPartial(ctx context.Context, enterpriseName string, pageToken string) (*androidmanagement.ListDevicesResponse, error) {
 	ret, err := g.mgmt.Enterprises.Devices.List(enterpriseName).Context(ctx).PageToken(pageToken).PageSize(100).Fields("nextPageToken", "devices/name").Do()
 	if err != nil {

server/mdm/android/service/androidmgmt/proxy_client.go

@@ -234,6 +234,17 @@ func (p *ProxyClient) EnterprisesDevicesIssueCommand(ctx context.Context, device
 	return op, nil
 }
 
+func (p *ProxyClient) EnterprisesDevicesOperationsGet(ctx context.Context, operationName string) (*androidmanagement.Operation, error) {
+	call := p.mgmt.Enterprises.Devices.Operations.Get(operationName).Context(ctx)
+	call.Header().Set("Authorization", "Bearer "+p.fleetServerSecret)
+	op, err := call.Do()
+	if err != nil {
+		// Wrapped with %w so callers can classify the googleapi.Error (not found, quota exceeded).
+		return nil, fmt.Errorf("getting operation %s: %w", operationName, err)
+	}
+	return op, nil
+}
+
 func (p *ProxyClient) EnterprisesDevicesListPartial(ctx context.Context, enterpriseName string, pageToken string) (*androidmanagement.ListDevicesResponse, error) {
 	call := p.mgmt.Enterprises.Devices.List(enterpriseName).Context(ctx).PageToken(pageToken).PageSize(100).Fields("nextPageToken", "devices/name")
 	call.Header().Set("Authorization", "Bearer "+p.fleetServerSecret)

server/mdm/android/service/pubsub.go

@@ -188,11 +188,12 @@ func (svc *Service) handlePubSubCommand(ctx context.Context, token string, rawDa
 	}
 
 	// Already-terminal rows. AMAPI may redeliver a notification at-least-once.
-	// For WIPE+acknowledged specifically, still re-run handleAndroidWipeAckUnenroll so transient DB
+	// For WIPE+acknowledged specifically, still re-run androidWipeAckUnenroll so transient DB
 	// failures on the original delivery recover on this retry.
 	if cmd.Status != string(android.MDMAndroidCommandStatusPending) {
 		if cmd.CommandType == string(android.MDMAndroidCommandTypeWipe) && cmd.Status == string(android.MDMAndroidCommandStatusAcknowledged) {
-			if err := svc.handleAndroidWipeAckUnenroll(ctx, cmd, messageID, publishTime); err != nil {
+			if err := androidWipeAckUnenroll(ctx, svc.fleetDS, svc.newActivity, cmd,
+				svc.pubSubDedupRecorder(ctx, messageID, publishTime)); err != nil {
 				return err
 			}
 		}
@@ -201,28 +202,10 @@ func (svc *Service) handlePubSubCommand(ctx context.Context, token string, rawDa
 		return nil
 	}
 
-	newStatus := string(android.MDMAndroidCommandStatusAcknowledged)
-	var errCode, errMsg *string
-	if op.Error != nil {
-		newStatus = string(android.MDMAndroidCommandStatusError)
-		code := googleStatusCode(op.Error.Code)
-		message := op.Error.Message
-		errCode = &code
-		errMsg = &message
-	}
-
-	if err := svc.fleetDS.UpdateMDMAndroidCommandStatus(ctx, cmd.CommandUUID, newStatus, errCode, errMsg); err != nil {
-		return ctxerr.Wrap(ctx, err, "update android command status from pub/sub")
-	}
-
-	// WIPE ack is the authoritative signal that the device has been wiped (BYO: work profile removed; COBO: full factory reset). Flip
-	// host_mdm.enrolled to 0 here rather than waiting on a separate STATUS_REPORT / ENROLLMENT with state=DELETED, which AMAPI does
-	// not reliably send for a factory-reset COBO device (the agent is gone, nothing left to phone home). For BYO the DELETED
-	// notification typically arrives and is now a no-op because we already flipped state.
-	if cmd.CommandType == string(android.MDMAndroidCommandTypeWipe) && newStatus == string(android.MDMAndroidCommandStatusAcknowledged) {
-		if err := svc.handleAndroidWipeAckUnenroll(ctx, cmd, messageID, publishTime); err != nil {
-			return err
-		}
+	newStatus, errCode, errMsg := androidOperationTerminalState(&op)
+	if err := setAndroidCommandTerminalState(ctx, svc.fleetDS, svc.newActivity, cmd, newStatus, errCode, errMsg,
+		svc.pubSubDedupRecorder(ctx, messageID, publishTime)); err != nil {
+		return err
 	}
 
 	svc.logger.InfoContext(ctx, "android pub/sub COMMAND processed",
@@ -234,11 +217,57 @@ func (svc *Service) handlePubSubCommand(ctx context.Context, token string, rawDa
 	return nil
 }
 
-// handleAndroidWipeAckUnenroll runs after a successful WIPE ack: flips host_mdm.enrolled, clears host_mdm_actions for BYO (so the
+// androidOperationTerminalState maps a done AMAPI Operation to the terminal status to write on the
+// mdm_android_commands row, plus the error code/message to record. A nil Operation.Error means the
+// device executed the command successfully; a populated one means AMAPI or the device rejected it.
+func androidOperationTerminalState(op *androidmanagement.Operation) (status string, errCode, errMsg *string) {
+	if op.Error == nil {
+		return string(android.MDMAndroidCommandStatusAcknowledged), nil, nil
+	}
+	code := googleStatusCode(op.Error.Code)
+	message := op.Error.Message
+	return string(android.MDMAndroidCommandStatusError), &code, &message
+}
+
+// setAndroidCommandTerminalState moves a pending mdm_android_commands row to a terminal status and runs
+// the post-WIPE-ack side effects. Shared by the Pub/Sub COMMAND handler and the command reconciler cron
+// so the two paths cannot drift. onUnenrolled is passed through to androidWipeAckUnenroll; see its doc
+// comment.
+func setAndroidCommandTerminalState(ctx context.Context, ds fleet.Datastore, newActivityFn fleet.NewActivityFunc,
+	cmd *android.MDMAndroidCommand, status string, errCode, errMsg *string, onUnenrolled func(hostID uint),
+) error {
+	// WIPE ack is the authoritative signal that the device has been wiped (BYO: work profile removed; COBO: full factory reset). Flip
+	// host_mdm.enrolled to 0 here rather than waiting on a separate STATUS_REPORT / ENROLLMENT with state=DELETED, which AMAPI does
+	// not reliably send for a factory-reset COBO device (the agent is gone, nothing left to phone home). For BYO the DELETED
+	// notification typically arrives and is now a no-op because we already flipped state.
+	//
+	// This runs before the status write, not after: androidWipeAckUnenroll is idempotent, so a failure
+	// here leaving the row pending is recoverable (Pub/Sub redelivers, and the reconciler cron only
+	// selects pending rows). Writing the status first would strand a row as acknowledged with its side
+	// effects never applied, which the reconciler could never pick up again.
+	if cmd.CommandType == string(android.MDMAndroidCommandTypeWipe) && status == string(android.MDMAndroidCommandStatusAcknowledged) {
+		if err := androidWipeAckUnenroll(ctx, ds, newActivityFn, cmd, onUnenrolled); err != nil {
+			return err
+		}
+	}
+
+	if err := ds.UpdateMDMAndroidCommandStatus(ctx, cmd.CommandUUID, status, errCode, errMsg); err != nil {
+		return ctxerr.Wrap(ctx, err, "update android command status")
+	}
+	return nil
+}
+
+// androidWipeAckUnenroll runs after a successful WIPE ack: flips host_mdm.enrolled, clears host_mdm_actions for BYO (so the
 // "Wiped" badge does not stick on a host whose only the work profile was removed), and emits mdm_unenrolled if state actually
 // changed. Returns errors so Pub/Sub retries on transient DB failures.
-func (svc *Service) handleAndroidWipeAckUnenroll(ctx context.Context, cmd *android.MDMAndroidCommand, messageID, publishTime string) error {
-	ah, err := svc.ds.AndroidHostLiteByHostUUID(ctx, cmd.HostUUID)
+//
+// onUnenrolled, when non-nil, runs only if this call actually flipped the host to unenrolled. The Pub/Sub
+// path uses it to record dedup state for the notification that drove the wipe; the reconciler cron passes
+// nil because it has no Pub/Sub message to dedup against.
+func androidWipeAckUnenroll(ctx context.Context, ds fleet.Datastore, newActivityFn fleet.NewActivityFunc,
+	cmd *android.MDMAndroidCommand, onUnenrolled func(hostID uint),
+) error {
+	ah, err := ds.AndroidHostLiteByHostUUID(ctx, cmd.HostUUID)
 	if err != nil {
 		return ctxerr.Wrap(ctx, err, "android wipe-ack unenroll: lookup host by uuid")
 	}
@@ -248,11 +277,11 @@ func (svc *Service) handleAndroidWipeAckUnenroll(ctx context.Context, cmd *andro
 
 	// BYO needs host_mdm_actions cleared so IsWiped() returns false post-ack -- only the work
 	// profile was removed, not the device. COBO leaves wipe_ref intact so the "Wiped" badge sticks.
-	if err := clearAndroidBYOWipeRef(ctx, svc.fleetDS, ah.Host.ID); err != nil {
+	if err := clearAndroidBYOWipeRef(ctx, ds, ah.Host.ID); err != nil {
 		return ctxerr.Wrap(ctx, err, "android wipe-ack unenroll: clear byo wipe-ref")
 	}
 
-	didUnenroll, err := svc.fleetDS.SetAndroidHostUnenrolled(ctx, ah.Host.ID)
+	didUnenroll, err := ds.SetAndroidHostUnenrolled(ctx, ah.Host.ID)
 	if err != nil {
 		return ctxerr.Wrap(ctx, err, "android wipe-ack unenroll: set host_mdm unenrolled")
 	}
@@ -277,13 +306,15 @@ func (svc *Service) handleAndroidWipeAckUnenroll(ctx context.Context, cmd *andro
 	// it here means a STATUS_REPORT published before the wipe but delivered afterwards (Pub/Sub
 	// is unordered) is dropped as stale by handlePubSubStatusReport, so it cannot re-enroll a
 	// device that was just wiped. Only done when this delivery actually flipped state.
-	svc.recordPubSubProcessed(ctx, ah.Host.ID, messageID, pubSubEventTime("", publishTime))
+	if onUnenrolled != nil {
+		onUnenrolled(ah.Host.ID)
+	}
 
 	displayName := ""
-	if hosts, herr := svc.fleetDS.ListHostsLiteByIDs(ctx, []uint{ah.Host.ID}); herr == nil && len(hosts) == 1 && hosts[0] != nil {
+	if hosts, herr := ds.ListHostsLiteByIDs(ctx, []uint{ah.Host.ID}); herr == nil && len(hosts) == 1 && hosts[0] != nil {
 		displayName = hosts[0].DisplayName()
 	}
-	if err := svc.newActivity(ctx, nil, fleet.ActivityTypeMDMUnenrolled{
+	if err := newActivityFn(ctx, nil, fleet.ActivityTypeMDMUnenrolled{
 		HostID:           ah.Host.ID,
 		HostDisplayName:  displayName,
 		InstalledFromDEP: false,
@@ -416,6 +447,15 @@ func (svc *Service) recordPubSubProcessed(ctx context.Context, hostID uint, mess
 	}
 }
 
+// pubSubDedupRecorder builds the onUnenrolled callback for androidWipeAckUnenroll from a COMMAND
+// notification's envelope. The COMMAND payload carries no device timestamp, so publishTime is the
+// only available event time.
+func (svc *Service) pubSubDedupRecorder(ctx context.Context, messageID, publishTime string) func(hostID uint) {
+	return func(hostID uint) {
+		svc.recordPubSubProcessed(ctx, hostID, messageID, pubSubEventTime("", publishTime))
+	}
+}
+
 func (svc *Service) handlePubSubStatusReport(ctx context.Context, token string, rawData []byte, messageID, publishTime string) error {
 	err := svc.authenticatePubSub(ctx, token)
 	if err != nil {

server/mdm/android/service/reconcile_commands.go

@@ -0,0 +1,175 @@
+package service
+
+import (
+	"context"
+	"log/slog"
+	"time"
+
+	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/fleetdm/fleet/v4/server/mdm/android"
+	"github.com/fleetdm/fleet/v4/server/mdm/android/service/androidmgmt"
+)
+
+const (
+	// androidCommandReconcileMinAge is how long a command must sit in the pending status before we poll
+	// AMAPI for it. Pub/Sub delivers within seconds in normal operation, so anything younger than this is
+	// still expected to resolve on its own and polling it would only burn AMAPI quota.
+	androidCommandReconcileMinAge = 24 * time.Hour
+
+	// androidCommandReconcileNotFoundGrace is how long we keep waiting on a command whose Operation AMAPI
+	// no longer knows about before declaring it failed. AMAPI drops Operation resources it has finished
+	// with, and it also 404s for a device that was deleted, so a NotFound on its own does not tell us the
+	// command is dead -- AMAPI may still be holding it (e.g. a WIPE waiting for the device to come back
+	// online). Once the row is older than GCP Pub/Sub's maximum retention no notification can arrive
+	// anymore, so at that point the row can only be stuck and marking it failed is what unsticks the host.
+	androidCommandReconcileNotFoundGrace = 7 * 24 * time.Hour
+
+	// androidCommandReconcileBatchSize bounds how many commands (and therefore AMAPI calls) a single run
+	// makes. Combined with the rate limit below this caps a run at ~10 minutes of polling. Rows that don't
+	// fit are picked up by the next run: they are ordered oldest-first, so the most stuck ones go first.
+	androidCommandReconcileBatchSize = 500
+
+	// androidCommandReconcileCallsPerMinute is the AMAPI request rate the reconciler paces itself to, to
+	// stay well under the per-project request budget shared with the rest of Fleet's AMAPI traffic.
+	androidCommandReconcileCallsPerMinute = 50
+
+	// googleStatusCodeNotFound is google.rpc.Code NOT_FOUND, recorded on rows we fail because AMAPI no
+	// longer has the Operation.
+	googleStatusCodeNotFound = 5
+)
+
+// ReconcileAndroidCommands recovers Android MDM commands whose Pub/Sub COMMAND notification never
+// arrived (Fleet's push endpoint down longer than GCP's retention, a subscription misconfiguration, a
+// Google Cloud incident). Without this, such a command sits in mdm_android_commands.status='pending'
+// forever, the host reads as perpetually pending lock/wipe/clear-passcode, and the admin cannot
+// re-issue it. AMAPI's operations.get is the authoritative source for a command's outcome and, unlike
+// Apple's and Windows' equivalents, needs neither the device to come back online nor AMAPI to re-send
+// anything.
+func ReconcileAndroidCommands(ctx context.Context, ds fleet.Datastore, logger *slog.Logger, licenseKey string, newActivityFn fleet.NewActivityFunc) error {
+	appConfig, err := ds.AppConfig(ctx)
+	if err != nil {
+		return ctxerr.Wrap(ctx, err, "get app config")
+	}
+	if !appConfig.MDM.AndroidEnabledAndConfigured {
+		return nil
+	}
+
+	client := newAMAPIClient(ctx, logger, licenseKey)
+
+	// Set the authentication secret for proxy client usage (a no-op for the Google client, which
+	// authenticates from its own env var and has no such asset). Without it every AMAPI call on the proxy
+	// path is rejected, so say so loudly rather than letting the run burn through the batch on 401s.
+	assets, err := ds.GetAllMDMConfigAssetsByName(ctx, []fleet.MDMAssetName{fleet.MDMAssetAndroidFleetServerSecret}, nil)
+	switch {
+	case err != nil:
+		logger.WarnContext(ctx, "could not read the android fleet server secret; AMAPI calls will fail if this Fleet uses the proxy client", "err", err)
+	default:
+		asset, ok := assets[fleet.MDMAssetAndroidFleetServerSecret]
+		if !ok || len(asset.Value) == 0 {
+			logger.WarnContext(ctx, "no android fleet server secret stored; AMAPI calls will fail if this Fleet uses the proxy client")
+		} else if err := client.SetAuthenticationSecret(string(asset.Value)); err != nil {
+			return ctxerr.Wrap(ctx, err, "set android fleet server secret")
+		}
+	}
+
+	return reconcileAndroidCommands(ctx, ds, client, logger, newActivityFn, time.Now().UTC(),
+		time.Minute/androidCommandReconcileCallsPerMinute)
+}
+
+// reconcileAndroidCommands is the testable core of ReconcileAndroidCommands. now anchors both the
+// pending-age cutoff and the NotFound grace period; callInterval is the delay between AMAPI calls.
+func reconcileAndroidCommands(ctx context.Context, ds fleet.Datastore, client androidmgmt.Client, logger *slog.Logger,
+	newActivityFn fleet.NewActivityFunc, now time.Time, callInterval time.Duration,
+) error {
+	cmds, err := ds.ListPendingMDMAndroidCommands(ctx, now.Add(-androidCommandReconcileMinAge), androidCommandReconcileBatchSize)
+	if err != nil {
+		return ctxerr.Wrap(ctx, err, "list pending android commands for reconcile")
+	}
+	if len(cmds) == 0 {
+		return nil
+	}
+
+	ticker := time.NewTicker(callInterval)
+	defer ticker.Stop()
+
+	var resolved, stillRunning int
+	for i, cmd := range cmds {
+		// Pace ourselves between AMAPI calls, but don't pay the delay before the first one.
+		if i > 0 {
+			select {
+			case <-ticker.C:
+			case <-ctx.Done():
+				return ctxerr.Wrap(ctx, ctx.Err(), "android command reconcile interrupted")
+			}
+		}
+
+		op, err := client.EnterprisesDevicesOperationsGet(ctx, cmd.OperationName)
+		switch {
+		case androidmgmt.IsTooManyRequestsError(err):
+			// Out of AMAPI quota. Stop the run rather than hammering a rate-limited API; the remaining rows
+			// stay pending and the next run resumes with them (oldest first).
+			logger.WarnContext(ctx, "android command reconcile hit AMAPI quota, stopping run",
+				"command_uuid", cmd.CommandUUID, "resolved", resolved, "remaining", len(cmds)-i)
+			return ctxerr.Wrap(ctx, err, "android command reconcile exceeded AMAPI quota")
+
+		case androidmgmt.IsAuthenticationError(err):
+			// Bad or missing credentials, or Fleet lost access to the enterprise. Every remaining call
+			// would be rejected the same way, so stop instead of working through the batch on errors that
+			// say nothing about the individual commands.
+			logger.ErrorContext(ctx, "android command reconcile rejected by AMAPI, stopping run",
+				"command_uuid", cmd.CommandUUID, "resolved", resolved, "remaining", len(cmds)-i, "err", err)
+			return ctxerr.Wrap(ctx, err, "android command reconcile rejected by AMAPI")
+
+		case androidmgmt.IsNotFoundError(err):
+			age := now.Sub(cmd.CreatedAt)
+			if age < androidCommandReconcileNotFoundGrace {
+				logger.DebugContext(ctx, "android command operation not found in AMAPI, still within grace period",
+					"command_uuid", cmd.CommandUUID, "operation_name", cmd.OperationName, "age", age)
+				stillRunning++
+				continue
+			}
+			errCode := googleStatusCode(googleStatusCodeNotFound)
+			errMsg := "Fleet did not receive a result for this command and Google no longer has a record of it."
+			// nil dedup recorder: this path is driven by the cron, not a Pub/Sub notification, so there is
+			// no messageId or publish time to record.
+			if err := setAndroidCommandTerminalState(ctx, ds, newActivityFn, cmd,
+				string(android.MDMAndroidCommandStatusError), &errCode, &errMsg, nil); err != nil {
+				logger.ErrorContext(ctx, "failed to fail android command with unknown operation",
+					"command_uuid", cmd.CommandUUID, "err", err)
+				ctxerr.Handle(ctx, err)
+				continue
+			}
+			resolved++
+			logger.InfoContext(ctx, "android command operation unknown to AMAPI past grace period, marked error",
+				"command_uuid", cmd.CommandUUID, "operation_name", cmd.OperationName, "age", age)
+
+		case err != nil:
+			// Transient AMAPI/network failure for this one command. Keep going: the rest of the batch is
+			// independent, and this row is retried on the next run.
+			logger.ErrorContext(ctx, "failed to get android command operation from AMAPI",
+				"command_uuid", cmd.CommandUUID, "operation_name", cmd.OperationName, "err", err)
+			ctxerr.Handle(ctx, ctxerr.Wrap(ctx, err, "get android command operation from AMAPI"))
+
+		case !op.Done:
+			// Still queued at AMAPI (e.g. the device has not come online yet). Leave it pending.
+			stillRunning++
+
+		default:
+			status, errCode, errMsg := androidOperationTerminalState(op)
+			if err := setAndroidCommandTerminalState(ctx, ds, newActivityFn, cmd, status, errCode, errMsg, nil); err != nil {
+				logger.ErrorContext(ctx, "failed to apply reconciled android command status",
+					"command_uuid", cmd.CommandUUID, "status", status, "err", err)
+				ctxerr.Handle(ctx, err)
+				continue
+			}
+			resolved++
+			logger.InfoContext(ctx, "android command reconciled from AMAPI",
+				"command_uuid", cmd.CommandUUID, "command_type", cmd.CommandType, "new_status", status)
+		}
+	}
+
+	logger.DebugContext(ctx, "android command reconcile complete",
+		"checked", len(cmds), "resolved", resolved, "still_running", stillRunning)
+	return nil
+}

server/mdm/android/service/reconcile_commands_test.go

@@ -0,0 +1,351 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/fleetdm/fleet/v4/server/mdm/android"
+	android_mock "github.com/fleetdm/fleet/v4/server/mdm/android/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/api/androidmanagement/v1"
+	"google.golang.org/api/googleapi"
+)
+
+// reconcileNow is the fixed "current time" the reconcile tests run at, so command ages are exact.
+var reconcileNow = time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC)
+
+// reconcileTestCallInterval keeps the reconciler's AMAPI pacing out of the tests' wall-clock time.
+const reconcileTestCallInterval = time.Nanosecond
+
+// pendingCommandForReconcile builds a pending command row of the given type, created age ago.
+func pendingCommandForReconcile(cmdUUID, cmdType string, age time.Duration) *android.MDMAndroidCommand {
+	return &android.MDMAndroidCommand{
+		CommandUUID:   cmdUUID,
+		HostUUID:      "host-uuid-" + cmdUUID,
+		OperationName: "enterprises/E/devices/D/operations/" + cmdUUID,
+		CommandType:   cmdType,
+		Status:        string(android.MDMAndroidCommandStatusPending),
+		CreatedAt:     reconcileNow.Add(-age),
+	}
+}
+
+// newReconcileFixture wires a mock datastore and AMAPI client for the reconciler. cmds is what
+// ListPendingMDMAndroidCommands returns; the caller shapes the client's operations.get behavior.
+func newReconcileFixture(t *testing.T, cmds ...*android.MDMAndroidCommand) (*AndroidMockDS, *android_mock.Client, *slog.Logger) {
+	t.Helper()
+	mockDS := InitCommonDSMocks()
+	mockDS.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
+		return &fleet.AppConfig{MDM: fleet.MDM{AndroidEnabledAndConfigured: true}}, nil
+	}
+	mockDS.ListPendingMDMAndroidCommandsFunc = func(ctx context.Context, createdBefore time.Time, limit int) ([]*android.MDMAndroidCommand, error) {
+		require.Equal(t, reconcileNow.Add(-androidCommandReconcileMinAge), createdBefore)
+		require.Equal(t, androidCommandReconcileBatchSize, limit)
+		return cmds, nil
+	}
+	client := &android_mock.Client{}
+	client.InitCommonMocks()
+	// Discard log output: these tests assert on datastore effects, not on log lines.
+	return mockDS, client, slog.New(slog.NewTextHandler(io.Discard, nil))
+}
+
+// googleAPIError builds the *googleapi.Error shape the AMAPI clients return, so the reconciler's
+// status-code classification is exercised the way it is in production.
+func googleAPIError(code int, message string) error {
+	return &googleapi.Error{Code: code, Message: message}
+}
+
+func TestReconcileAndroidCommands(t *testing.T) {
+	t.Run("done operation transitions the row to its terminal status", func(t *testing.T) {
+		for _, tc := range []struct {
+			name           string
+			opError        *androidmanagement.Status
+			expectedStatus string
+			expectedCode   string
+			expectedMsg    string
+		}{
+			{
+				name:           "no error means the device executed the command",
+				opError:        nil,
+				expectedStatus: string(android.MDMAndroidCommandStatusAcknowledged),
+			},
+			{
+				name:           "populated error records the google.rpc code and message",
+				opError:        &androidmanagement.Status{Code: 13, Message: "device does not support LOCK"},
+				expectedStatus: string(android.MDMAndroidCommandStatusError),
+				expectedCode:   "13",
+				expectedMsg:    "device does not support LOCK",
+			},
+		} {
+			t.Run(tc.name, func(t *testing.T) {
+				cmd := pendingCommandForReconcile("cmd-done", string(android.MDMAndroidCommandTypeLock), 48*time.Hour)
+				mockDS, client, logger := newReconcileFixture(t, cmd)
+				client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) {
+					require.Equal(t, cmd.OperationName, operationName)
+					return &androidmanagement.Operation{Name: operationName, Done: true, Error: tc.opError}, nil
+				}
+				var gotStatus string
+				var gotCode, gotMsg *string
+				mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error {
+					require.Equal(t, cmd.CommandUUID, commandUUID)
+					gotStatus, gotCode, gotMsg = status, errorCode, errorMessage
+					return nil
+				}
+
+				require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval))
+
+				require.True(t, mockDS.UpdateMDMAndroidCommandStatusFuncInvoked)
+				assert.Equal(t, tc.expectedStatus, gotStatus)
+				if tc.expectedCode == "" {
+					assert.Nil(t, gotCode)
+					assert.Nil(t, gotMsg)
+				} else {
+					require.NotNil(t, gotCode)
+					require.NotNil(t, gotMsg)
+					assert.Equal(t, tc.expectedCode, *gotCode)
+					assert.Equal(t, tc.expectedMsg, *gotMsg)
+				}
+			})
+		}
+	})
+
+	t.Run("operation still running is left pending", func(t *testing.T) {
+		cmd := pendingCommandForReconcile("cmd-running", string(android.MDMAndroidCommandTypeWipe), 48*time.Hour)
+		mockDS, client, logger := newReconcileFixture(t, cmd)
+		client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) {
+			return &androidmanagement.Operation{Name: operationName, Done: false}, nil
+		}
+		mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error {
+			t.Fatalf("a command AMAPI is still working on must not be transitioned")
+			return nil
+		}
+
+		require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval))
+		require.False(t, mockDS.UpdateMDMAndroidCommandStatusFuncInvoked)
+	})
+
+	t.Run("unknown operation inside the grace period is left pending", func(t *testing.T) {
+		// AMAPI 404s for an operation it has already discarded, but a notification can still arrive while
+		// the row is younger than Pub/Sub's retention, so we keep waiting.
+		cmd := pendingCommandForReconcile("cmd-404-young", string(android.MDMAndroidCommandTypeLock),
+			androidCommandReconcileNotFoundGrace-time.Hour)
+		mockDS, client, logger := newReconcileFixture(t, cmd)
+		client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) {
+			return nil, googleAPIError(http.StatusNotFound, "Requested entity was not found.")
+		}
+		mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error {
+			t.Fatalf("a command still inside the not-found grace period must not be transitioned")
+			return nil
+		}
+
+		require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval))
+		require.False(t, mockDS.UpdateMDMAndroidCommandStatusFuncInvoked)
+	})
+
+	t.Run("unknown operation past the grace period is marked error", func(t *testing.T) {
+		// Past Pub/Sub's retention no notification can arrive anymore, so the row can only be stuck.
+		cmd := pendingCommandForReconcile("cmd-404-old", string(android.MDMAndroidCommandTypeLock),
+			androidCommandReconcileNotFoundGrace+time.Hour)
+		mockDS, client, logger := newReconcileFixture(t, cmd)
+		client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) {
+			return nil, googleAPIError(http.StatusNotFound, "Requested entity was not found.")
+		}
+		var gotStatus string
+		var gotCode, gotMsg *string
+		mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error {
+			gotStatus, gotCode, gotMsg = status, errorCode, errorMessage
+			return nil
+		}
+
+		require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval))
+
+		require.True(t, mockDS.UpdateMDMAndroidCommandStatusFuncInvoked)
+		assert.Equal(t, string(android.MDMAndroidCommandStatusError), gotStatus)
+		require.NotNil(t, gotCode)
+		assert.Equal(t, "5", *gotCode, "google.rpc.Code NOT_FOUND")
+		require.NotNil(t, gotMsg)
+		assert.NotEmpty(t, *gotMsg)
+	})
+
+	t.Run("acknowledged WIPE runs the unenroll side effects", func(t *testing.T) {
+		const hostID uint = 42
+		cmd := pendingCommandForReconcile("cmd-wipe", string(android.MDMAndroidCommandTypeWipe), 48*time.Hour)
+		mockDS, client, logger := newReconcileFixture(t, cmd)
+		client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) {
+			return &androidmanagement.Operation{Name: operationName, Done: true}, nil
+		}
+		mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error {
+			return nil
+		}
+		mockDS.AndroidHostLiteByHostUUIDFunc = func(ctx context.Context, hostUUID string) (*fleet.AndroidHost, error) {
+			require.Equal(t, cmd.HostUUID, hostUUID)
+			return &fleet.AndroidHost{Host: &fleet.Host{ID: hostID, UUID: hostUUID}}, nil
+		}
+		// BYO: the work profile was removed, so host_mdm_actions must be cleared for the "Wiped" badge to drop.
+		mockDS.GetHostMDMFunc = func(ctx context.Context, id uint) (*fleet.HostMDM, error) {
+			return &fleet.HostMDM{IsPersonalEnrollment: true}, nil
+		}
+		mockDS.ClearHostMDMActionsFunc = func(ctx context.Context, id uint) error { return nil }
+		mockDS.SetAndroidHostUnenrolledFunc = func(ctx context.Context, id uint) (bool, error) {
+			require.Equal(t, hostID, id)
+			return true, nil
+		}
+		mockDS.ListHostsLiteByIDsFunc = func(ctx context.Context, ids []uint) ([]*fleet.Host, error) {
+			return []*fleet.Host{{ID: hostID, Hostname: "wiped-host"}}, nil
+		}
+		var activities []fleet.ActivityDetails
+		newActivity := func(_ context.Context, _ *fleet.User, details fleet.ActivityDetails) error {
+			activities = append(activities, details)
+			return nil
+		}
+
+		require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, newActivity, reconcileNow, reconcileTestCallInterval))
+
+		require.True(t, mockDS.ClearHostMDMActionsFuncInvoked, "BYO wipe must clear host_mdm_actions")
+		require.True(t, mockDS.SetAndroidHostUnenrolledFuncInvoked, "a wiped host must be flipped to unenrolled")
+		require.Len(t, activities, 1)
+		require.IsType(t, fleet.ActivityTypeMDMUnenrolled{}, activities[0])
+	})
+
+	t.Run("a failed WIPE side effect leaves the command pending so the next run retries it", func(t *testing.T) {
+		// The reconciler only ever selects pending rows, so writing the terminal status before the
+		// unenroll side effect succeeds would strand the host: acknowledged, still enrolled, and never
+		// looked at again.
+		cmd := pendingCommandForReconcile("cmd-wipe-transient", string(android.MDMAndroidCommandTypeWipe), 48*time.Hour)
+		mockDS, client, logger := newReconcileFixture(t, cmd)
+		client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) {
+			return &androidmanagement.Operation{Name: operationName, Done: true}, nil
+		}
+		mockDS.AndroidHostLiteByHostUUIDFunc = func(ctx context.Context, hostUUID string) (*fleet.AndroidHost, error) {
+			return &fleet.AndroidHost{Host: &fleet.Host{ID: 55, UUID: hostUUID}}, nil
+		}
+		mockDS.GetHostMDMFunc = func(ctx context.Context, id uint) (*fleet.HostMDM, error) {
+			return &fleet.HostMDM{IsPersonalEnrollment: false}, nil
+		}
+		mockDS.SetAndroidHostUnenrolledFunc = func(ctx context.Context, id uint) (bool, error) {
+			return false, errors.New("simulated transient DB connection drop")
+		}
+		mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error {
+			t.Fatalf("the command must stay pending when its wipe side effect fails")
+			return nil
+		}
+
+		require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval))
+		require.False(t, mockDS.UpdateMDMAndroidCommandStatusFuncInvoked)
+	})
+
+	t.Run("errored WIPE does not unenroll the host", func(t *testing.T) {
+		cmd := pendingCommandForReconcile("cmd-wipe-failed", string(android.MDMAndroidCommandTypeWipe), 48*time.Hour)
+		mockDS, client, logger := newReconcileFixture(t, cmd)
+		client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) {
+			return &androidmanagement.Operation{
+				Name:  operationName,
+				Done:  true,
+				Error: &androidmanagement.Status{Code: 13, Message: "wipe failed"},
+			}, nil
+		}
+		mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error {
+			require.Equal(t, string(android.MDMAndroidCommandStatusError), status)
+			return nil
+		}
+
+		require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval))
+		require.False(t, mockDS.SetAndroidHostUnenrolledFuncInvoked, "a failed wipe must leave the host enrolled")
+	})
+
+	t.Run("a failure on one command does not stop the rest of the batch", func(t *testing.T) {
+		failing := pendingCommandForReconcile("cmd-transient", string(android.MDMAndroidCommandTypeLock), 48*time.Hour)
+		updateFailing := pendingCommandForReconcile("cmd-update-fails", string(android.MDMAndroidCommandTypeLock), 48*time.Hour)
+		succeeding := pendingCommandForReconcile("cmd-ok", string(android.MDMAndroidCommandTypeLock), 48*time.Hour)
+		mockDS, client, logger := newReconcileFixture(t, failing, updateFailing, succeeding)
+		client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) {
+			if operationName == failing.OperationName {
+				return nil, errors.New("simulated transient network failure")
+			}
+			return &androidmanagement.Operation{Name: operationName, Done: true}, nil
+		}
+		var updated []string
+		mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error {
+			if commandUUID == updateFailing.CommandUUID {
+				return errors.New("simulated transient DB failure")
+			}
+			updated = append(updated, commandUUID)
+			return nil
+		}
+
+		require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval))
+		require.Equal(t, []string{succeeding.CommandUUID}, updated,
+			"the reconciler must keep going past both an AMAPI failure and a DB failure")
+	})
+
+	t.Run("AMAPI quota error stops the run and surfaces an error", func(t *testing.T) {
+		first := pendingCommandForReconcile("cmd-quota", string(android.MDMAndroidCommandTypeLock), 48*time.Hour)
+		second := pendingCommandForReconcile("cmd-after-quota", string(android.MDMAndroidCommandTypeLock), 48*time.Hour)
+		mockDS, client, logger := newReconcileFixture(t, first, second)
+		var calls int
+		client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) {
+			calls++
+			return nil, googleAPIError(http.StatusTooManyRequests, "Quota exceeded")
+		}
+
+		err := reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval)
+		require.Error(t, err)
+		require.Equal(t, 1, calls, "the run must stop at the first quota error instead of hammering AMAPI")
+		require.False(t, mockDS.UpdateMDMAndroidCommandStatusFuncInvoked)
+	})
+
+	t.Run("AMAPI rejecting our credentials stops the run and surfaces an error", func(t *testing.T) {
+		// A missing or stale Fleet server secret, lost access to the enterprise, or (on the proxy path)
+		// fleetdm.com having no record of the enterprise, rejects every call identically -- working
+		// through the batch would only produce noise.
+		for _, statusCode := range []int{http.StatusUnauthorized, http.StatusForbidden} {
+			first := pendingCommandForReconcile("cmd-rejected", string(android.MDMAndroidCommandTypeLock), 48*time.Hour)
+			second := pendingCommandForReconcile("cmd-after-rejected", string(android.MDMAndroidCommandTypeLock), 48*time.Hour)
+			mockDS, client, logger := newReconcileFixture(t, first, second)
+			var calls int
+			client.EnterprisesDevicesOperationsGetFunc = func(ctx context.Context, operationName string) (*androidmanagement.Operation, error) {
+				calls++
+				return nil, googleAPIError(statusCode, "rejected")
+			}
+
+			err := reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval)
+			require.Error(t, err, "status code %d", statusCode)
+			require.Equal(t, 1, calls, "status code %d must stop the run at the first rejection", statusCode)
+			require.False(t, mockDS.UpdateMDMAndroidCommandStatusFuncInvoked,
+				"a rejected call says nothing about the command, so nothing may be marked failed")
+		}
+	})
+
+	t.Run("nothing pending makes no AMAPI calls", func(t *testing.T) {
+		mockDS, client, logger := newReconcileFixture(t)
+
+		require.NoError(t, reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval))
+		require.False(t, client.EnterprisesDevicesOperationsGetFuncInvoked)
+	})
+
+	t.Run("a datastore failure surfaces so the cron run is marked failed", func(t *testing.T) {
+		mockDS, client, logger := newReconcileFixture(t)
+		mockDS.ListPendingMDMAndroidCommandsFunc = func(ctx context.Context, createdBefore time.Time, limit int) ([]*android.MDMAndroidCommand, error) {
+			return nil, errors.New("simulated DB outage")
+		}
+
+		err := reconcileAndroidCommands(t.Context(), &mockDS.DataStore, client, logger, noopNewActivity, reconcileNow, reconcileTestCallInterval)
+		require.ErrorContains(t, err, "simulated DB outage")
+	})
+
+	t.Run("android MDM turned off skips the run entirely", func(t *testing.T) {
+		mockDS, _, logger := newReconcileFixture(t)
+		mockDS.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
+			return &fleet.AppConfig{MDM: fleet.MDM{AndroidEnabledAndConfigured: false}}, nil
+		}
+
+		require.NoError(t, ReconcileAndroidCommands(t.Context(), &mockDS.DataStore, logger, "", noopNewActivity))
+		require.False(t, mockDS.ListPendingMDMAndroidCommandsFuncInvoked)
+	})
+}

server/mock/datastore_mock.go

@@ -1988,6 +1988,8 @@ type GetMDMAndroidCommandByOperationNameFunc func(ctx context.Context, operation
 
 type UpdateMDMAndroidCommandStatusFunc func(ctx context.Context, commandUUID string, status string, errorCode *string, errorMessage *string) error
 
+type ListPendingMDMAndroidCommandsFunc func(ctx context.Context, createdBefore time.Time, limit int) ([]*android.MDMAndroidCommand, error)
+
 type LockHostViaAndroidMDMFunc func(ctx context.Context, host *fleet.Host, cmd *android.MDMAndroidCommand) error
 
 type WipeHostViaAndroidMDMFunc func(ctx context.Context, host *fleet.Host, cmd *android.MDMAndroidCommand) error
@@ -5251,6 +5253,9 @@ type DataStore struct {
 	UpdateMDMAndroidCommandStatusFunc        UpdateMDMAndroidCommandStatusFunc
 	UpdateMDMAndroidCommandStatusFuncInvoked bool
 
+	ListPendingMDMAndroidCommandsFunc        ListPendingMDMAndroidCommandsFunc
+	ListPendingMDMAndroidCommandsFuncInvoked bool
+
 	LockHostViaAndroidMDMFunc        LockHostViaAndroidMDMFunc
 	LockHostViaAndroidMDMFuncInvoked bool
 
@@ -12602,6 +12607,13 @@ func (s *DataStore) UpdateMDMAndroidCommandStatus(ctx context.Context, commandUU
 	return s.UpdateMDMAndroidCommandStatusFunc(ctx, commandUUID, status, errorCode, errorMessage)
 }
 
+func (s *DataStore) ListPendingMDMAndroidCommands(ctx context.Context, createdBefore time.Time, limit int) ([]*android.MDMAndroidCommand, error) {
+	s.mu.Lock()
+	s.ListPendingMDMAndroidCommandsFuncInvoked = true
+	s.mu.Unlock()
+	return s.ListPendingMDMAndroidCommandsFunc(ctx, createdBefore, limit)
+}
+
 func (s *DataStore) LockHostViaAndroidMDM(ctx context.Context, host *fleet.Host, cmd *android.MDMAndroidCommand) error {
 	s.mu.Lock()
 	s.LockHostViaAndroidMDMFuncInvoked = true

tools/hangar/frontend/src/lib/fleetctlCrons.ts

@@ -105,6 +105,12 @@ export const CRONS: CronInfo[] = [
     interval: "1h",
     note: "Reconciles Android device existence with Google AMAPI.",
   },
+  {
+    name: "mdm_android_command_reconciler",
+    group: "mdm",
+    interval: "24h",
+    note: "Resolves stuck Android MDM commands via AMAPI operations.get.",
+  },
 
   // ---------- activity / maintenance ----------
   {

website/api/controllers/android-proxy/get-android-device-operation.js

@@ -0,0 +1,111 @@
+module.exports = {
+
+
+  friendlyName: 'Get android device operation',
+
+
+  description: 'Gets a long-running operation for a device of an Android enterprise. Fleet servers poll this to recover the outcome of an Android MDM command (Lock, Wipe, Clear passcode) whose Pub/Sub COMMAND notification never arrived.',
+
+
+  inputs: {
+    androidEnterpriseId: {
+      type: 'string',
+      required: true,
+    },
+    deviceId: {
+      type: 'string',
+      required: true,
+    },
+    operationId: {
+      type: 'string',
+      required: true,
+    },
+  },
+
+
+  exits: {
+    success: { description: 'The operation for a device of an Android enterprise was successfully retrieved.' },
+    missingAuthHeader: { description: 'This request was missing an authorization header.', responseType: 'unauthorized'},
+    unauthorized: { description: 'Invalid authentication token.', responseType: 'unauthorized'},
+    // Unlike the other android-proxy actions, this one reserves 404 for a single meaning: the Android
+    // management API has no record of this operation. The Fleet server treats that as evidence the
+    // command can never complete and eventually marks it failed, so nothing about *this website's*
+    // records may return a 404. A missing AndroidEnterprise row and a loss of access to the enterprise
+    // both mean "we cannot answer for this enterprise" -- a 403, which the Fleet server classifies as
+    // an authorization failure and stops its whole reconciler run on.
+    enterpriseNotAccessible: { description: 'No Android enterprise found for this Fleet server, or Fleet is not authorized to manage it.', statusCode: 403 },
+    operationNotFound: { description: 'The specified operation does not exist in this Android enterprise', responseType: 'notFound' },
+    // The Fleet server classifies this status code to know it should stop polling and wait for the next
+    // reconciler run, so the Android management API's 429 has to survive the trip through this proxy.
+    tooManyRequests: { description: 'The Android management API rate limit was exceeded.', statusCode: 429 },
+  },
+
+
+  fn: async function ({ androidEnterpriseId, deviceId, operationId }) {
+
+    // Extract fleetServerSecret from the Authorization header
+    let authHeader = this.req.get('authorization');
+    let fleetServerSecret;
+
+    if (authHeader && authHeader.startsWith('Bearer')) {
+      fleetServerSecret = authHeader.replace('Bearer', '').trim();
+    } else {
+      throw 'missingAuthHeader';
+    }
+
+    // Authenticate this request
+    let thisAndroidEnterprise = await AndroidEnterprise.findOne({
+      androidEnterpriseId: androidEnterpriseId
+    });
+
+    // Return a 403 (not a 404) if no records are found -- see the note on the exits above.
+    if (!thisAndroidEnterprise) {
+      throw 'enterpriseNotAccessible';
+    }
+    // Return an unauthorized response if the provided secret does not match.
+    if (thisAndroidEnterprise.fleetServerSecret !== fleetServerSecret) {
+      throw 'unauthorized';
+    }
+
+
+    // Get the shared Google API auth client with the getAndroidManagementAuthorizationClient helper.
+    // Note: we are doing this outside of the sails.helpers.flow.build() so any errors related to the website's credentials returned by the helper are not intercepted.
+    let androidManagementAuthClient = await sails.helpers.androidProxy.getAndroidManagementAuthorizationClient();
+
+    // Get the operation for this device.
+    // Note: We're using sails.helpers.flow.build here to handle any errors that occur using google's node library.
+    let getOperationResponse = await sails.helpers.flow.build(async () => {
+      let { google } = require('googleapis');
+      let androidManagementConnection = google.androidmanagement({version: 'v1', auth: androidManagementAuthClient});
+      // [?]: https://googleapis.dev/nodejs/googleapis/latest/androidmanagement/classes/Resource$Enterprises$Devices$Operations.html#get
+      let getOperationResult = await androidManagementConnection.enterprises.devices.operations.get({
+        name: `enterprises/${androidEnterpriseId}/devices/${deviceId}/operations/${operationId}`,
+      });
+      return getOperationResult.data;
+    }).intercept({status: 429}, ()=>{
+      // If the Android management API returns a 429 response, log an additional warning that will trigger a help-p1 alert.
+      // Note: the error object is deliberately left out of this log -- gaxios errors carry the request
+      // config, including the Authorization header used to call Google.
+      sails.log.warn(`p1: Android management API rate limit exceeded! (When getting a device operation for Android enterprise ${androidEnterpriseId}.)`);
+      // Pass the 429 through to the Fleet server rather than collapsing it into a 500, so its reconciler
+      // can tell rate limiting apart from a generic failure.
+      return 'tooManyRequests';
+    }).intercept({status: 403}, ()=>{
+      // If the Android management API returns a 403 response, return an enterpriseNotAccessible (403) response to the Fleet server.
+      return 'enterpriseNotAccessible';
+    }).intercept({status: 404}, ()=>{
+      // If the Android management API returns a 404 response, return an operationNotFound (notFound) response to the Fleet server.
+      // The Fleet server treats this as "Google no longer has a record of this command".
+      return 'operationNotFound';
+    }).intercept((err)=>{
+      return new Error(`When attempting to get a device operation for an Android enterprise (${androidEnterpriseId}), an error occurred. Error: ${require('util').inspect(err)}`);
+    });
+
+
+    // Return the operation data back to the Fleet server.
+    return getOperationResponse;
+
+  }
+
+
+};

website/config/routes.js

@@ -1438,6 +1438,7 @@ module.exports.routes = {
   'PATCH /api/android/v1/enterprises/:androidEnterpriseId/policies/:policyId': { action: 'android-proxy/modify-android-policies', csrf: false },
   'DELETE /api/android/v1/enterprises/:androidEnterpriseId': { action: 'android-proxy/delete-one-android-enterprise', csrf: false },
   'GET /api/android/v1/enterprises/:androidEnterpriseId/devices/:deviceId': { action: 'android-proxy/get-android-device' },
+  'GET /api/android/v1/enterprises/:androidEnterpriseId/devices/:deviceId/operations/:operationId': { action: 'android-proxy/get-android-device-operation' },
   'GET /api/android/v1/enterprises/:androidEnterpriseId/devices': { action: 'android-proxy/get-android-devices' },
   'DELETE /api/android/v1/enterprises/:androidEnterpriseId/devices/:deviceId': { action: 'android-proxy/delete-android-device', csrf: false },
   'PATCH /api/android/v1/enterprises/:androidEnterpriseId/devices/:deviceId': { action: 'android-proxy/modify-android-device', csrf: false },