Deduplicate Android MDM Pub/Sub deliveries and protect against reordering (#49792)

**Related issue:** Resolves #43502
This commit is contained in:
Dante Catalfamo
2026-08-06 17:24:54 -04:00
committed by GitHub
parent e29210e6f6
commit 3c8df41762
14 changed files with 1003 additions and 38 deletions
+1
View File
@@ -0,0 +1 @@
- Added deduplication and out-of-order protection to the Android MDM Pub/Sub notification handler. Duplicate deliveries from Google Pub/Sub no longer re-run the setup experience or emit duplicate activities, and a stale device-deleted notification arriving after a re-enrollment no longer leaves the host stuck showing unenrolled.
+123
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"slices"
"strings"
"time"
"unicode/utf8"
"github.com/fleetdm/fleet/v4/server/contexts/ctxdb"
@@ -576,6 +577,128 @@ UPDATE host_mdm
return rows > 0, nil
}
// GetAndroidPubSubDedupState returns the last-processed Google Pub/Sub messageId
// and AMAPI event timestamp recorded for the host, used by the AMAPI notification
// handler to drop duplicate (same messageId) and stale (older timestamp)
// deliveries. When the android_devices row exists but nothing has been recorded
// yet, it returns an empty messageId and nil eventTime with no error. When no
// android_devices row exists for the host, it returns a NotFound error.
func (ds *Datastore) GetAndroidPubSubDedupState(ctx context.Context, hostID uint) (messageID string, eventTime *time.Time, err error) {
var state struct {
MessageID *string `db:"last_pubsub_message_id"`
EventTime *time.Time `db:"last_pubsub_event_time"`
}
err = sqlx.GetContext(ctx, ds.reader(ctx), &state,
`SELECT last_pubsub_message_id, last_pubsub_event_time FROM android_devices WHERE host_id = ?`, hostID)
switch {
case errors.Is(err, sql.ErrNoRows):
return "", nil, ctxerr.Wrap(ctx, notFound("AndroidDevice").WithID(hostID), "get android pubsub dedup state")
case err != nil:
return "", nil, ctxerr.Wrap(ctx, err, "get android pubsub dedup state")
}
return ptr.ValOrZero(state.MessageID), state.EventTime, nil
}
// SetAndroidPubSubDedupState records the last-processed Google Pub/Sub messageId
// and AMAPI event timestamp for the host after a notification is handled
// successfully. Returns a NotFound error when no android_devices row matches
// hostID, so a missing row surfaces (via the caller's log) instead of silently
// dropping dedup state.
//
// An empty messageID or nil eventTime leaves that column at its previous value
// rather than clearing it. A notification that carries no usable timestamp says
// nothing about ordering, so overwriting the recorded baseline with NULL would
// disable staleness protection for the host until some later message happened to
// carry a parseable timestamp. The columns only ever move forward.
func (ds *Datastore) SetAndroidPubSubDedupState(ctx context.Context, hostID uint, messageID string, eventTime *time.Time) error {
// clientFoundRows is set on the DSN, so RowsAffected below counts matched rows, not
// changed rows — a write that preserves both columns still reports 1 for an existing row.
result, err := ds.writer(ctx).ExecContext(ctx, `
UPDATE android_devices
SET last_pubsub_message_id = IF(? = '', last_pubsub_message_id, ?),
last_pubsub_event_time = CASE
WHEN ? IS NULL THEN last_pubsub_event_time
WHEN last_pubsub_event_time IS NULL OR ? > last_pubsub_event_time THEN ?
ELSE last_pubsub_event_time
END
WHERE host_id = ?`,
messageID, messageID, eventTime, eventTime, eventTime, hostID)
if err != nil {
return ctxerr.Wrap(ctx, err, "set android pubsub dedup state")
}
rows, err := result.RowsAffected()
if err != nil {
return ctxerr.Wrap(ctx, err, "get rows affected for set android pubsub dedup state")
}
if rows == 0 {
return ctxerr.Wrap(ctx, notFound("AndroidDevice").WithID(hostID), "set android pubsub dedup state")
}
return nil
}
// SetAndroidHostEnrolled flips host_mdm back to enrolled for an Android host that
// is currently marked unenrolled. This recovers a host that was wrongly unenrolled
// by an out-of-order DELETED delivery: a live device sending a STATUS_REPORT is by
// definition still managed. It is a no-op (returns false) when the host is already
// enrolled or has no host_mdm row, so it is safe to call on every STATUS_REPORT. It
// intentionally does not re-run enrollment side effects (setup experience, cert
// templates, team assignment) — those belong to the ENROLLMENT path.
//
// It preserves the existing is_personal_enrollment classification rather than
// recomputing it: the triggering STATUS_REPORT payload may omit Ownership, which
// would otherwise misclassify a COBO (company-owned) host as personal.
func (ds *Datastore) SetAndroidHostEnrolled(ctx context.Context, hostID uint) (bool, error) {
// Fast path: this is called on every STATUS_REPORT, but almost always the host is
// already enrolled and there is nothing to do. Check that with a cheap read before
// opening a write transaction. The transaction below re-reads authoritatively, so a
// stale replica read here at worst causes a redundant (still-correct) transaction or
// defers recovery to the next report.
var enrolled bool
switch err := sqlx.GetContext(ctx, ds.reader(ctx), &enrolled,
`SELECT enrolled FROM host_mdm WHERE host_id = ?`, hostID); {
case errors.Is(err, sql.ErrNoRows):
return false, nil
case err != nil:
return false, ctxerr.Wrap(ctx, err, "check android host_mdm enrolled state")
case enrolled:
return false, nil
}
appCfg, err := ds.AppConfig(ctx)
if err != nil {
return false, ctxerr.Wrap(ctx, err, "set android host enrolled get app config")
}
var didEnroll bool
err = ds.withTx(ctx, func(tx sqlx.ExtContext) error {
var current struct {
Enrolled bool `db:"enrolled"`
IsPersonalEnrollment bool `db:"is_personal_enrollment"`
}
err := sqlx.GetContext(ctx, tx, &current,
`SELECT enrolled, is_personal_enrollment FROM host_mdm WHERE host_id = ?`, hostID)
switch {
case errors.Is(err, sql.ErrNoRows):
// No host_mdm row yet; leave enrollment to the ENROLLMENT path.
return nil
case err != nil:
return ctxerr.Wrap(ctx, err, "get android host_mdm enrolled state")
case current.Enrolled:
// Already enrolled: nothing to recover.
return nil
}
if err := upsertAndroidHostMDMInfoDB(ctx, tx, appCfg.ServerSettings.ServerURL, !current.IsPersonalEnrollment, true, hostID); err != nil {
return ctxerr.Wrap(ctx, err, "re-enroll android host_mdm info")
}
didEnroll = true
return nil
})
if err != nil {
return false, err
}
return didEnroll, nil
}
func upsertAndroidHostMDMInfoDB(ctx context.Context, tx sqlx.ExtContext, serverURL string, companyOwned, enrolled bool, hostID uint) error {
result, err := tx.ExecContext(ctx, `
INSERT INTO mobile_device_management_solutions (name, server_url) VALUES (?, ?)
+126
View File
@@ -61,6 +61,8 @@ func TestAndroid(t *testing.T) {
{"NewAndroidHostWithIdP", testNewAndroidHostWithIdP},
{"AndroidBYODDetection", testAndroidBYODDetection},
{"SetAndroidHostUnenrolled", testSetAndroidHostUnenrolled},
{"SetAndroidHostEnrolled", testSetAndroidHostEnrolled},
{"AndroidPubSubDedupState", testAndroidPubSubDedupState},
{"BulkSetAndroidHostsUnenrolled", testBulkSetAndroidHostsUnenrolled},
{"InsertAndGetAndroidAppConfiguration", testInsertAndGetAndroidAppConfiguration},
{"UpdateAndroidAppConfiguration", testUpdateAndroidAppConfiguration},
@@ -3393,6 +3395,130 @@ func testAndroidBYODDetection(t *testing.T, ds *Datastore) {
}
// NEW TEST: verify single-host unenroll updates host_mdm correctly
func testSetAndroidHostEnrolled(t *testing.T, ds *Datastore) {
appCfg, err := ds.AppConfig(testCtx())
require.NoError(t, err)
appCfg.ServerSettings.ServerURL = "https://mdm.example.com"
require.NoError(t, ds.SaveAppConfig(testCtx(), appCfg))
// Create a BYO Android host (companyOwned=false) -> enrolled host_mdm row.
esid := "enterprise-" + uuid.NewString()
res, err := ds.NewAndroidHost(testCtx(), createAndroidHost(esid), false)
require.NoError(t, err)
// Already enrolled: no-op, returns false.
didEnroll, err := ds.SetAndroidHostEnrolled(testCtx(), res.Host.ID)
require.NoError(t, err)
require.False(t, didEnroll, "SetAndroidHostEnrolled must be a no-op when the host is already enrolled")
// Unenroll, then recover.
unenrolled, err := ds.SetAndroidHostUnenrolled(testCtx(), res.Host.ID)
require.NoError(t, err)
require.True(t, unenrolled)
didEnroll, err = ds.SetAndroidHostEnrolled(testCtx(), res.Host.ID)
require.NoError(t, err)
require.True(t, didEnroll, "SetAndroidHostEnrolled must restore enrollment for an unenrolled host")
hostMDM, err := ds.GetHostMDM(testCtx(), res.Host.ID)
require.NoError(t, err)
require.True(t, hostMDM.Enrolled, "host_mdm.enrolled must be restored to 1")
require.Equal(t, "https://mdm.example.com", hostMDM.ServerURL, "server_url must be restored")
require.True(t, hostMDM.IsPersonalEnrollment, "BYO recovery must preserve is_personal_enrollment")
// Calling again is a no-op.
didEnroll, err = ds.SetAndroidHostEnrolled(testCtx(), res.Host.ID)
require.NoError(t, err)
require.False(t, didEnroll)
// Unknown host has no host_mdm row: no-op, no error.
didEnroll, err = ds.SetAndroidHostEnrolled(testCtx(), 999999)
require.NoError(t, err)
require.False(t, didEnroll)
// COBO recovery must preserve is_personal_enrollment=0 even though the recovery does
// not know the ownership (it is derived from the existing row, not the status payload).
coboESID := "enterprise-cobo-" + uuid.NewString()
cobo, err := ds.NewAndroidHost(testCtx(), createAndroidHost(coboESID), true /* companyOwned */)
require.NoError(t, err)
coboMDM, err := ds.GetHostMDM(testCtx(), cobo.Host.ID)
require.NoError(t, err)
require.False(t, coboMDM.IsPersonalEnrollment, "fresh COBO enrollment is not a personal enrollment")
unenrolled, err = ds.SetAndroidHostUnenrolled(testCtx(), cobo.Host.ID)
require.NoError(t, err)
require.True(t, unenrolled)
didEnroll, err = ds.SetAndroidHostEnrolled(testCtx(), cobo.Host.ID)
require.NoError(t, err)
require.True(t, didEnroll)
coboMDM, err = ds.GetHostMDM(testCtx(), cobo.Host.ID)
require.NoError(t, err)
require.True(t, coboMDM.Enrolled)
require.False(t, coboMDM.IsPersonalEnrollment, "COBO recovery must not reclassify the host as personal")
}
func testAndroidPubSubDedupState(t *testing.T, ds *Datastore) {
esid := "enterprise-" + uuid.NewString()
res, err := ds.NewAndroidHost(testCtx(), createAndroidHost(esid), false)
require.NoError(t, err)
hostID := res.Host.ID
// Fresh host: no recorded state.
messageID, eventTime, err := ds.GetAndroidPubSubDedupState(testCtx(), hostID)
require.NoError(t, err)
require.Empty(t, messageID)
require.Nil(t, eventTime)
// Record a messageId + event time.
t1 := time.Now().UTC().Truncate(time.Microsecond)
require.NoError(t, ds.SetAndroidPubSubDedupState(testCtx(), hostID, "msg-1", &t1))
messageID, eventTime, err = ds.GetAndroidPubSubDedupState(testCtx(), hostID)
require.NoError(t, err)
require.Equal(t, "msg-1", messageID)
require.NotNil(t, eventTime)
require.WithinDuration(t, t1, *eventTime, time.Millisecond)
// Overwrite with a newer message.
t2 := t1.Add(time.Hour)
require.NoError(t, ds.SetAndroidPubSubDedupState(testCtx(), hostID, "msg-2", &t2))
messageID, eventTime, err = ds.GetAndroidPubSubDedupState(testCtx(), hostID)
require.NoError(t, err)
require.Equal(t, "msg-2", messageID)
require.WithinDuration(t, t2, *eventTime, time.Millisecond)
// A nil event time records the messageId but preserves the timestamp baseline. Clearing
// it to NULL would disable staleness protection for the host until some later message
// happened to carry a parseable timestamp.
require.NoError(t, ds.SetAndroidPubSubDedupState(testCtx(), hostID, "msg-3", nil))
messageID, eventTime, err = ds.GetAndroidPubSubDedupState(testCtx(), hostID)
require.NoError(t, err)
require.Equal(t, "msg-3", messageID)
require.NotNil(t, eventTime, "a nil event time must not clear the recorded baseline")
require.WithinDuration(t, t2, *eventTime, time.Millisecond)
// An empty messageId advances only the timestamp — this is how ReconcileAndroidDevices
// records an out-of-band unenroll, which has no Pub/Sub message of its own.
t3 := t2.Add(time.Hour)
require.NoError(t, ds.SetAndroidPubSubDedupState(testCtx(), hostID, "", &t3))
messageID, eventTime, err = ds.GetAndroidPubSubDedupState(testCtx(), hostID)
require.NoError(t, err)
require.Equal(t, "msg-3", messageID, "an empty messageId must not clear the recorded messageId")
require.WithinDuration(t, t3, *eventTime, time.Millisecond)
// Writing the same values again still reports the row as found (clientFoundRows), so it
// must not be mistaken for a missing android_devices row.
require.NoError(t, ds.SetAndroidPubSubDedupState(testCtx(), hostID, "msg-3", &t3))
// Unknown host -> NotFound (both get and set).
_, _, err = ds.GetAndroidPubSubDedupState(testCtx(), 999999)
require.True(t, fleet.IsNotFound(err), "expected NotFound for unknown host, got %v", err)
err = ds.SetAndroidPubSubDedupState(testCtx(), 999999, "msg-x", &t2)
require.True(t, fleet.IsNotFound(err), "set on a missing android_devices row must surface NotFound, got %v", err)
}
func testSetAndroidHostUnenrolled(t *testing.T, ds *Datastore) {
// Set a non-empty server URL so initial enrolled row has data to clear
appCfg, err := ds.AppConfig(testCtx())
@@ -0,0 +1,27 @@
package tables
import (
"database/sql"
"fmt"
)
func init() {
MigrationClient.AddMigration(Up_20260806210232, Down_20260806210232)
}
func Up_20260806210232(tx *sql.Tx) error {
// Track the last-processed Google Pub/Sub message per Android device so the
// AMAPI notification handler can deduplicate at-least-once redeliveries
// (same messageId) and drop out-of-order deliveries (older event timestamp).
if _, err := tx.Exec(`ALTER TABLE android_devices
ADD COLUMN last_pubsub_message_id VARCHAR(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
ADD COLUMN last_pubsub_event_time TIMESTAMP(6) NULL DEFAULT NULL`); err != nil {
return fmt.Errorf("add pubsub dedup columns to android_devices: %w", err)
}
return nil
}
func Down_20260806210232(tx *sql.Tx) error {
return nil
}
@@ -0,0 +1,47 @@
package tables
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestUp_20260806210232(t *testing.T) {
db := applyUpToPrev(t)
// Create a host and its android_devices row before the migration.
res, err := db.Exec(`INSERT INTO hosts (hostname, uuid, platform, team_id, osquery_host_id, node_key,
detail_updated_at, label_updated_at, policy_updated_at)
VALUES ('android1', 'uuid1', 'android', NULL, 'oq1', 'nk1', '2026-01-01', '2026-01-01', '2026-01-01')`)
require.NoError(t, err)
hostID, err := res.LastInsertId()
require.NoError(t, err)
_, err = db.Exec(`INSERT INTO android_devices (host_id, device_id, enterprise_specific_id) VALUES (?, 'd1', 'esid1')`, hostID)
require.NoError(t, err)
// Apply migration.
applyNext(t, db)
// Existing rows have NULL for both new columns.
var messageID *string
require.NoError(t, db.Get(&messageID, `SELECT last_pubsub_message_id FROM android_devices WHERE device_id = 'd1'`))
require.Nil(t, messageID)
var eventTime *string
require.NoError(t, db.Get(&eventTime, `SELECT last_pubsub_event_time FROM android_devices WHERE device_id = 'd1'`))
require.Nil(t, eventTime)
// The columns are writable and round-trip.
_, err = db.Exec(`UPDATE android_devices
SET last_pubsub_message_id = 'msg-123', last_pubsub_event_time = '2026-07-22 10:00:00.000000'
WHERE device_id = 'd1'`)
require.NoError(t, err)
require.NoError(t, db.Get(&messageID, `SELECT last_pubsub_message_id FROM android_devices WHERE device_id = 'd1'`))
require.NotNil(t, messageID)
require.Equal(t, "msg-123", *messageID)
require.NoError(t, db.Get(&eventTime, `SELECT last_pubsub_event_time FROM android_devices WHERE device_id = 'd1'`))
require.NotNil(t, eventTime)
}
File diff suppressed because one or more lines are too long
+6
View File
@@ -2726,6 +2726,12 @@ func (ds *Datastore) markAllPendingVPPInstallsAsFailedForHost(ctx context.Contex
return nil, nil, ctxerr.New(ctx, fmt.Sprintf("softwareType %s not supported", softwareType))
}
// The activities returned to the caller are derived solely from failedCmds, which
// is scoped to still-pending installs (verification_failed_at IS NULL AND
// verification_at IS NULL AND canceled = 0). This makes the function idempotent for
// the Android DELETED path: a duplicate Pub/Sub DELETED delivery finds those rows
// already marked failed, so the SELECT returns an empty set and no duplicate
// failed-install activities are emitted.
const loadFailedCmdsStmt = `
SELECT
command_uuid
+13
View File
@@ -3892,6 +3892,19 @@ type AndroidDatastore interface {
AppConfig(ctx context.Context) (*AppConfig, error)
BulkSetAndroidHostsUnenrolled(ctx context.Context) error
SetAndroidHostUnenrolled(ctx context.Context, hostID uint) (bool, error)
// SetAndroidHostEnrolled flips host_mdm back to enrolled for an Android host
// that is currently marked unenrolled, recovering a host wrongly unenrolled by
// an out-of-order Pub/Sub DELETED delivery. Returns false (no-op) when the host
// is already enrolled or has no host_mdm row. It preserves the existing
// is_personal_enrollment classification.
SetAndroidHostEnrolled(ctx context.Context, hostID uint) (bool, error)
// GetAndroidPubSubDedupState returns the last-processed Google Pub/Sub messageId
// and AMAPI event timestamp recorded for the host, for dropping duplicate and
// stale AMAPI notification deliveries.
GetAndroidPubSubDedupState(ctx context.Context, hostID uint) (messageID string, eventTime *time.Time, err error)
// SetAndroidPubSubDedupState records the last-processed Google Pub/Sub messageId
// and AMAPI event timestamp for the host after a notification is handled.
SetAndroidPubSubDedupState(ctx context.Context, hostID uint, messageID string, eventTime *time.Time) error
DeleteMDMConfigAssetsByName(ctx context.Context, assetNames []MDMAssetName) error
GetAllMDMConfigAssetsByName(ctx context.Context, assetNames []MDMAssetName,
queryerContext sqlx.QueryerContext) (map[MDMAssetName]MDMConfigAsset, error)
+6
View File
@@ -19,4 +19,10 @@ const (
type PubSubMessage struct {
Attributes map[string]string `json:"attributes"`
Data string `json:"data"`
// MessageID and PublishTime are set by Google Pub/Sub on the push envelope as
// siblings of Attributes/Data. MessageID is stable across at-least-once
// redeliveries of the same message; PublishTime is an RFC3339 timestamp used as
// a staleness fallback when the AMAPI payload carries no event timestamp.
MessageID string `json:"messageId"`
PublishTime string `json:"publishTime"`
}
@@ -7,6 +7,7 @@ import (
"net/http"
"os"
"testing"
"time"
"github.com/fleetdm/fleet/v4/server/authz"
"github.com/fleetdm/fleet/v4/server/config"
@@ -237,6 +238,15 @@ func InitCommonDSMocks() *AndroidMockDS {
ds.Store.UpdateTeamIDOnAndroidDevicesFunc = func(ctx context.Context, hostUUIDs []string, teamID *uint) error {
return nil
}
ds.Store.GetAndroidPubSubDedupStateFunc = func(ctx context.Context, hostID uint) (string, *time.Time, error) {
return "", nil, nil
}
ds.Store.SetAndroidPubSubDedupStateFunc = func(ctx context.Context, hostID uint, messageID string, eventTime *time.Time) error {
return nil
}
ds.Store.SetAndroidHostEnrolledFunc = func(ctx context.Context, hostID uint) (bool, error) {
return false, nil
}
return &ds
}
+208 -36
View File
@@ -64,11 +64,11 @@ func (svc *Service) ProcessPubSubPush(ctx context.Context, token string, message
switch android.NotificationType(notificationType) {
case android.PubSubEnrollment:
return svc.handlePubSubEnrollment(ctx, token, rawData)
return svc.handlePubSubEnrollment(ctx, token, rawData, message.MessageID, message.PublishTime)
case android.PubSubStatusReport:
return svc.handlePubSubStatusReport(ctx, token, rawData)
return svc.handlePubSubStatusReport(ctx, token, rawData, message.MessageID, message.PublishTime)
case android.PubSubCommand:
return svc.handlePubSubCommand(ctx, token, rawData)
return svc.handlePubSubCommand(ctx, token, rawData, message.MessageID, message.PublishTime)
default:
// Ignore unknown notification types
svc.logger.DebugContext(ctx, "Ignoring PubSub notification type", "notification", notificationType)
@@ -146,7 +146,7 @@ func clearAndroidBYOWipeRef(ctx context.Context, ds fleet.Datastore, hostID uint
// notification to the Fleet row via operation_name and transition the mdm_android_commands row from pending to
// acknowledged or error. host_mdm_actions does not need updating: HostLockWipeStatus reads the row status string
// directly.
func (svc *Service) handlePubSubCommand(ctx context.Context, token string, rawData []byte) error {
func (svc *Service) handlePubSubCommand(ctx context.Context, token string, rawData []byte, messageID, publishTime string) error {
if err := svc.authenticatePubSub(ctx, token); err != nil {
return err
}
@@ -192,7 +192,7 @@ func (svc *Service) handlePubSubCommand(ctx context.Context, token string, rawDa
// failures on the original delivery recover on this retry.
if cmd.Status != string(android.MDMAndroidCommandStatusPending) {
if cmd.CommandType == string(android.MDMAndroidCommandTypeWipe) && cmd.Status == string(android.MDMAndroidCommandStatusAcknowledged) {
if err := svc.handleAndroidWipeAckUnenroll(ctx, cmd); err != nil {
if err := svc.handleAndroidWipeAckUnenroll(ctx, cmd, messageID, publishTime); err != nil {
return err
}
}
@@ -220,7 +220,7 @@ func (svc *Service) handlePubSubCommand(ctx context.Context, token string, rawDa
// not reliably send for a factory-reset COBO device (the agent is gone, nothing left to phone home). For BYO the DELETED
// notification typically arrives and is now a no-op because we already flipped state.
if cmd.CommandType == string(android.MDMAndroidCommandTypeWipe) && newStatus == string(android.MDMAndroidCommandStatusAcknowledged) {
if err := svc.handleAndroidWipeAckUnenroll(ctx, cmd); err != nil {
if err := svc.handleAndroidWipeAckUnenroll(ctx, cmd, messageID, publishTime); err != nil {
return err
}
}
@@ -237,7 +237,7 @@ func (svc *Service) handlePubSubCommand(ctx context.Context, token string, rawDa
// handleAndroidWipeAckUnenroll runs after a successful WIPE ack: flips host_mdm.enrolled, clears host_mdm_actions for BYO (so the
// "Wiped" badge does not stick on a host whose only the work profile was removed), and emits mdm_unenrolled if state actually
// changed. Returns errors so Pub/Sub retries on transient DB failures.
func (svc *Service) handleAndroidWipeAckUnenroll(ctx context.Context, cmd *android.MDMAndroidCommand) error {
func (svc *Service) handleAndroidWipeAckUnenroll(ctx context.Context, cmd *android.MDMAndroidCommand, messageID, publishTime string) error {
ah, err := svc.ds.AndroidHostLiteByHostUUID(ctx, cmd.HostUUID)
if err != nil {
return ctxerr.Wrap(ctx, err, "android wipe-ack unenroll: lookup host by uuid")
@@ -256,6 +256,7 @@ func (svc *Service) handleAndroidWipeAckUnenroll(ctx context.Context, cmd *andro
if err != nil {
return ctxerr.Wrap(ctx, err, "android wipe-ack unenroll: set host_mdm unenrolled")
}
if !didUnenroll {
// Already unenrolled (e.g. the API wrapper for BYO Unenroll already ran, a prior DELETED
// notification beat us, or a prior delivery flipped state and is now retrying). No state
@@ -263,9 +264,21 @@ func (svc *Service) handleAndroidWipeAckUnenroll(ctx context.Context, cmd *andro
// the tradeoff is no duplicate activity rows after a successful first delivery, at the cost
// of losing the activity in the rare "flip succeeded then activity failed" race. The state
// flip is what matters; the activity loss is detectable via logs.
//
// We also do NOT re-record dedup state here: a redelivery of an already-terminal wipe must
// not move last_pubsub_event_time backwards to the (older) wipe publish time if a newer
// notification has since been recorded.
return nil
}
// Advance the dedup event time to the wipe notification's publish time. This is the
// authoritative COBO unenroll signal (AMAPI does not reliably send DELETED for a
// factory-reset device), and the COMMAND envelope carries no device timestamp. Recording
// it here means a STATUS_REPORT published before the wipe but delivered afterwards (Pub/Sub
// is unordered) is dropped as stale by handlePubSubStatusReport, so it cannot re-enroll a
// device that was just wiped. Only done when this delivery actually flipped state.
svc.recordPubSubProcessed(ctx, ah.Host.ID, messageID, pubSubEventTime("", publishTime))
displayName := ""
if hosts, herr := svc.fleetDS.ListHostsLiteByIDs(ctx, []uint{ah.Host.ID}); herr == nil && len(hosts) == 1 && hosts[0] != nil {
displayName = hosts[0].DisplayName()
@@ -336,7 +349,74 @@ func googleStatusCode(code int64) string {
return fmt.Sprintf("%d", code)
}
func (svc *Service) handlePubSubStatusReport(ctx context.Context, token string, rawData []byte) error {
// pubSubEventTime derives the AMAPI event timestamp used for staleness comparison.
// It prefers the device's LastStatusReportTime (present on STATUS_REPORT and,
// usually, ENROLLMENT device payloads) and falls back to the Pub/Sub envelope
// publishTime. Returns nil when neither is a parseable RFC3339 timestamp, in which
// case the staleness check is skipped and only messageId dedup applies.
//
// Caveat: the two sources are different Google clocks (device status time vs.
// Pub/Sub publish time). ENROLLMENT payloads often omit LastStatusReportTime, so a
// comparison may end up device-time vs. publish-time. Both are Google-side and close
// in practice, so the risk of misordering is low, but callers should not assume
// same-clock semantics.
func pubSubEventTime(deviceTime, publishTime string) *time.Time {
for _, ts := range []string{deviceTime, publishTime} {
if ts == "" {
continue
}
if t, err := time.Parse(time.RFC3339, ts); err == nil {
return &t
}
}
return nil
}
// isDuplicateOrStalePubSub reports whether an AMAPI notification for hostID should
// be skipped because it is a redelivery (same messageId as the last processed) or
// arrived out of order (event timestamp older than the last processed). Google
// Pub/Sub gives at-least-once, unordered delivery, so both cases occur in normal
// operation. A host with no recorded state yet is never a duplicate.
func (svc *Service) isDuplicateOrStalePubSub(ctx context.Context, hostID uint, messageID string, eventTime *time.Time) (bool, error) {
// Force the primary: Pub/Sub redeliveries commonly arrive within seconds, inside the
// replica-lag window, and reading a stale (empty) row here would let the redelivery
// reprocess — defeating the dedup.
lastMessageID, lastEventTime, err := svc.ds.GetAndroidPubSubDedupState(ctxdb.RequirePrimary(ctx, true), hostID)
if err != nil {
if fleet.IsNotFound(err) {
return false, nil
}
return false, ctxerr.Wrap(ctx, err, "get android pubsub dedup state")
}
if messageID != "" && messageID == lastMessageID {
svc.logger.DebugContext(ctx, "skipping duplicate Android PubSub message", "host_id", hostID, "message_id", messageID)
return true, nil
}
if eventTime != nil && lastEventTime != nil && eventTime.Before(*lastEventTime) {
svc.logger.DebugContext(ctx, "skipping stale Android PubSub message", "host_id", hostID,
"message_id", messageID, "event_time", eventTime, "last_event_time", lastEventTime)
return true, nil
}
return false, nil
}
// recordPubSubProcessed stores the messageId and event timestamp of a
// successfully-handled notification so future duplicate/stale deliveries for the
// host are dropped. Failure is non-fatal: the message was already processed, and
// returning an error would trigger a Pub/Sub retry that reprocesses (and could
// re-emit) the same work. A missed record only weakens dedup for the narrow
// redelivery window.
func (svc *Service) recordPubSubProcessed(ctx context.Context, hostID uint, messageID string, eventTime *time.Time) {
if err := svc.ds.SetAndroidPubSubDedupState(ctx, hostID, messageID, eventTime); err != nil {
// Logged at Warn, not Error: a NotFound here means the android_devices row was deleted
// between resolving the host and this write (a benign host-deletion race), not a fault
// that needs alerting.
svc.logger.WarnContext(ctx, "failed to record Android PubSub dedup state",
"host_id", hostID, "message_id", messageID, "err", err)
}
}
func (svc *Service) handlePubSubStatusReport(ctx context.Context, token string, rawData []byte, messageID, publishTime string) error {
err := svc.authenticatePubSub(ctx, token)
if err != nil {
return err
@@ -353,6 +433,8 @@ func (svc *Service) handlePubSubStatusReport(ctx context.Context, token string,
return err
}
eventTime := pubSubEventTime(device.LastStatusReportTime, publishTime)
// NOTE: uncomment as needed, can be useful for debugging as the pubsub report
// can be very large - it is not practical to print so it saves it to a file,
// different names for all instances of the pubsub, and under an extension that
@@ -386,6 +468,16 @@ func (svc *Service) handlePubSubStatusReport(ctx context.Context, token string,
return ctxerr.Wrap(ctx, err, "get host for deleted android device")
}
if host != nil {
// Drop duplicate/out-of-order deliveries before touching enrollment state.
// This is what stops a stale DELETED (redelivered after a re-ENROLLMENT)
// from unenrolling a live host, and advancing the recorded event time here
// stops a later stale STATUS_REPORT from wrongly re-enrolling it.
if skip, err := svc.isDuplicateOrStalePubSub(ctx, host.Host.ID, messageID, eventTime); err != nil {
return err
} else if skip {
return nil
}
// Capture BYO-ness BEFORE flipping host_mdm.enrolled, then clear host_mdm_actions for BYO
// so the post-ack "Wiped" badge clears (BYO unenroll only wipes the work profile).
if err := clearAndroidBYOWipeRef(ctx, svc.fleetDS, host.Host.ID); err != nil {
@@ -412,6 +504,8 @@ func (svc *Service) handlePubSubStatusReport(ctx context.Context, token string,
}
}
svc.recordPubSubProcessed(ctx, host.Host.ID, messageID, eventTime)
if !didUnenroll {
return nil // Skip activity, if we didn't update the enrollment state.
}
@@ -441,8 +535,7 @@ func (svc *Service) handlePubSubStatusReport(ctx context.Context, token string,
svc.logger.DebugContext(ctx, "Device not found in Fleet. Perhaps it was deleted, "+
"but it is still connected via Android MDM. Re-enrolling", "device.name", device.Name,
"device.enterpriseSpecificId", device.HardwareInfo.EnterpriseSpecificId)
err = svc.enrollHost(ctx, &device)
if err != nil {
if _, err := svc.enrollHost(ctx, &device); err != nil {
svc.logger.DebugContext(ctx, "Error re-enrolling Android host", "data", rawData)
return ctxerr.Wrap(ctx, err, "re-enrolling deleted Android host")
}
@@ -458,15 +551,39 @@ func (svc *Service) handlePubSubStatusReport(ctx context.Context, token string,
device.HardwareInfo.EnterpriseSpecificId)
}
}
// Drop duplicate/out-of-order deliveries. A freshly re-enrolled host (host was
// nil above) has no recorded state, so this is a no-op for that case.
if skip, err := svc.isDuplicateOrStalePubSub(ctx, host.Host.ID, messageID, eventTime); err != nil {
return err
} else if skip {
return nil
}
err = svc.updateHost(ctx, &device, host, false)
if err != nil {
svc.logger.DebugContext(ctx, "Error updating Android host", "data", rawData)
return ctxerr.Wrap(ctx, err, "enrolling Android host")
}
// A live device sending a STATUS_REPORT is by definition still managed. If it is
// currently marked unenrolled (e.g. a stale DELETED slipped through before dedup
// state existed), restore enrollment so it does not stay stuck unenrolled until a
// fresh ENROLLMENT. The staleness check above prevents a stale STATUS_REPORT from
// re-enrolling a host that was legitimately unenrolled (including via a WIPE ack,
// whose unenroll path records the wipe's event time).
if didEnroll, err := svc.ds.SetAndroidHostEnrolled(ctx, host.Host.ID); err != nil {
return ctxerr.Wrap(ctx, err, "restore android host enrollment on status report")
} else if didEnroll {
svc.logger.InfoContext(ctx, "restored Android host enrollment from status report", "host_id", host.Host.ID)
}
err = svc.updateHostSoftware(ctx, &device, host)
if err != nil {
return ctxerr.Wrap(ctx, err, "updating Android host software")
}
svc.recordPubSubProcessed(ctx, host.Host.ID, messageID, eventTime)
return nil
}
@@ -509,7 +626,7 @@ func (svc *Service) updateHostSoftware(ctx context.Context, device *androidmanag
return nil
}
func (svc *Service) handlePubSubEnrollment(ctx context.Context, token string, rawData []byte) error {
func (svc *Service) handlePubSubEnrollment(ctx context.Context, token string, rawData []byte, messageID, publishTime string) error {
err := svc.authenticatePubSub(ctx, token)
if err != nil {
return err
@@ -527,6 +644,8 @@ func (svc *Service) handlePubSubEnrollment(ctx context.Context, token string, ra
return err
}
eventTime := pubSubEventTime(device.LastStatusReportTime, publishTime)
// Some deployments may report work profile removal under ENROLLMENT notifications.
// Detect DELETED here too and treat as unenrollment confirmation.
isDeleted := strings.ToUpper(device.AppliedState) == string(android.DeviceStateDeleted)
@@ -547,13 +666,21 @@ func (svc *Service) handlePubSubEnrollment(ctx context.Context, token string, ra
return ctxerr.Wrap(ctx, herr, "get host for deleted android device (ENROLLMENT)")
}
if host != nil {
// Drop duplicate/out-of-order deliveries before touching enrollment state.
if skip, err := svc.isDuplicateOrStalePubSub(ctx, host.Host.ID, messageID, eventTime); err != nil {
return err
} else if skip {
return nil
}
// Capture BYO-ness BEFORE flipping host_mdm.enrolled, then clear host_mdm_actions for BYO
// so the post-ack "Wiped" badge clears (BYO unenroll only wipes the work profile).
if err := clearAndroidBYOWipeRef(ctx, svc.fleetDS, host.Host.ID); err != nil {
return ctxerr.Wrap(ctx, err, "clear byo wipe-ref on DELETED state (ENROLLMENT)")
}
if _, err := svc.ds.SetAndroidHostUnenrolled(ctx, host.Host.ID); err != nil {
didUnenroll, err := svc.ds.SetAndroidHostUnenrolled(ctx, host.Host.ID)
if err != nil {
return ctxerr.Wrap(ctx, err, "set android host unenrolled on DELETED state (ENROLLMENT)")
}
@@ -572,6 +699,16 @@ func (svc *Service) handlePubSubEnrollment(ctx context.Context, token string, ra
}
}
svc.recordPubSubProcessed(ctx, host.Host.ID, messageID, eventTime)
if !didUnenroll {
// Already unenrolled (e.g. a DELETED delivered under STATUS_REPORT beat this one,
// or a redelivery that messageId dedup did not catch). Skip the activity so the
// feed does not gain a duplicate mdm_unenrolled row — same rule as the
// STATUS_REPORT DELETED branch.
return nil
}
var displayName, serial string
if hosts, herr := svc.fleetDS.ListHostsLiteByIDs(ctx, []uint{host.Host.ID}); herr == nil && len(hosts) == 1 && hosts[0] != nil {
displayName = hosts[0].DisplayName()
@@ -588,18 +725,49 @@ func (svc *Service) handlePubSubEnrollment(ctx context.Context, token string, ra
return nil
}
err = svc.enrollHost(ctx, &device)
// Drop duplicate ENROLLMENT deliveries before enrolling: a redelivered ENROLLMENT
// for an existing host would otherwise re-queue the setup-experience job (duplicate
// VPP installs and activities). A device brand-new to Fleet has no row to check
// against yet; its state is recorded below so a redelivery is caught.
// Force the primary so a redelivered ENROLLMENT sees a host that a prior delivery just
// created (and thus its recorded dedup state), instead of missing it on a lagging replica
// and re-queuing the setup experience.
existing, herr := svc.getExistingHost(ctxdb.RequirePrimary(ctx, true), &device)
if herr != nil {
return ctxerr.Wrap(ctx, herr, "getting existing Android host for enrollment dedup")
}
if existing != nil {
if skip, err := svc.isDuplicateOrStalePubSub(ctx, existing.Host.ID, messageID, eventTime); err != nil {
return err
} else if skip {
return nil
}
}
hostID, err := svc.enrollHost(ctx, &device)
if err != nil {
svc.logger.DebugContext(ctx, "Error enrolling Android host", "data", rawData)
return ctxerr.Wrap(ctx, err, "enrolling Android host")
}
// Record dedup state using the ID enrollHost resolved, rather than re-reading the host
// from the payload. A re-read can fail (replica lag, DB hiccup, a deploy returning 5xx)
// *after* enrollment and the setup-experience job have already run — the delivery would
// still be acked with no dedup state written, and the redelivery would re-queue the
// setup experience. That is the exact failure this dedup exists to prevent.
svc.recordPubSubProcessed(ctx, hostID, messageID, eventTime)
return nil
}
func (svc *Service) enrollHost(ctx context.Context, device *androidmanagement.Device) error {
// enrollHost enrolls (or re-enrolls) the device and returns the Fleet host ID of the
// resulting host. Returning the ID lets callers record follow-up state without a second
// lookup: a lookup that fails *after* enrollment has already run leaves the Pub/Sub
// delivery acked with that state unwritten, which is exactly the window a 5xx-inducing
// deploy or DB hiccup opens.
func (svc *Service) enrollHost(ctx context.Context, device *androidmanagement.Device) (uint, error) {
err := svc.validateDevice(ctx, device)
if err != nil {
return err
return 0, err
}
// Enqueue a job to send any necessary self-service software.
@@ -608,7 +776,7 @@ func (svc *Service) enrollHost(ctx context.Context, device *androidmanagement.De
// Device may already be present in Fleet if device user removed the MDM profile and then re-enrolled
host, err := svc.getExistingHost(ctx, device)
if err != nil {
return ctxerr.Wrap(ctx, err, "getting existing Android host")
return 0, ctxerr.Wrap(ctx, err, "getting existing Android host")
}
// TODO(mna): in the next iteration of Android work (as we're short on time
@@ -619,7 +787,7 @@ func (svc *Service) enrollHost(ctx context.Context, device *androidmanagement.De
var enrollmentTokenRequest enrollmentTokenRequest
err = json.Unmarshal([]byte(device.EnrollmentTokenData), &enrollmentTokenRequest)
if err != nil {
return ctxerr.Wrap(ctx, err, "unmarshalling enrollment token data")
return 0, ctxerr.Wrap(ctx, err, "unmarshalling enrollment token data")
}
if host != nil {
@@ -627,7 +795,7 @@ func (svc *Service) enrollHost(ctx context.Context, device *androidmanagement.De
"device.name", device.Name, "device.enterpriseSpecificId", device.HardwareInfo.EnterpriseSpecificId)
enrollSecret, err := svc.ds.VerifyEnrollSecret(ctx, enrollmentTokenRequest.EnrollSecret)
if err != nil && !fleet.IsNotFound(err) {
return ctxerr.Wrap(ctx, err, "verifying enroll secret")
return 0, ctxerr.Wrap(ctx, err, "verifying enroll secret")
}
if err == nil {
host.TeamID = enrollSecret.GetTeamID()
@@ -644,11 +812,14 @@ func (svc *Service) enrollHost(ctx context.Context, device *androidmanagement.De
if enrollmentTokenRequest.IdpUUID != "" {
if err := svc.ds.AssociateHostMDMIdPAccount(ctx, host.Host.UUID, enrollmentTokenRequest.IdpUUID); err != nil {
return ctxerr.Wrap(ctx, err, "updating IdP account on re-enrollment")
return 0, ctxerr.Wrap(ctx, err, "updating IdP account on re-enrollment")
}
}
return svc.updateHost(ctx, device, host, true)
if err := svc.updateHost(ctx, device, host, true); err != nil {
return 0, err
}
return host.Host.ID, nil
}
// Device is new to Fleet
@@ -890,24 +1061,25 @@ func setAndroidHostUUID(host *fleet.AndroidHost, device *androidmanagement.Devic
host.Device.EnterpriseSpecificID = ptr.String(uuidKey)
}
func (svc *Service) addNewHost(ctx context.Context, device *androidmanagement.Device) error {
// addNewHost inserts a host that is new to Fleet and returns its Fleet host ID.
func (svc *Service) addNewHost(ctx context.Context, device *androidmanagement.Device) (uint, error) {
// Validate before dereferencing device.SoftwareInfo/MemoryInfo/HardwareInfo
// below. enrollHost already validates before dispatching here, but this keeps
// addNewHost self-contained so it cannot panic if called from another path,
// matching updateHost.
if err := svc.validateDevice(ctx, device); err != nil {
return err
return 0, err
}
var enrollmentTokenRequest enrollmentTokenRequest
err := json.Unmarshal([]byte(device.EnrollmentTokenData), &enrollmentTokenRequest)
if err != nil {
return ctxerr.Wrap(ctx, err, "unmarshilling enrollment token data")
return 0, ctxerr.Wrap(ctx, err, "unmarshilling enrollment token data")
}
enrollSecret, err := svc.ds.VerifyEnrollSecret(ctx, enrollmentTokenRequest.EnrollSecret)
if err != nil {
return ctxerr.Wrap(ctx, err, "verifying enroll secret")
return 0, ctxerr.Wrap(ctx, err, "verifying enroll secret")
}
// If the device was previously known restore the last-known team instead of the enrollment secret's default.
@@ -922,14 +1094,14 @@ func (svc *Service) addNewHost(ctx context.Context, device *androidmanagement.De
deviceID, err := svc.getDeviceID(ctx, device)
if err != nil {
return ctxerr.Wrap(ctx, err, "getting device ID")
return 0, ctxerr.Wrap(ctx, err, "getting device ID")
}
gigsTotalDiskSpace, gigsDiskSpaceAvailable, percentDiskSpaceAvailable := svc.calculateAndroidStorageMetrics(ctx, device, false)
computerName, err := getComputerName(ctx, svc.fleetDS, device, nil, "", enrollmentTokenRequest.IdpUUID)
if err != nil {
return ctxerr.Wrap(ctx, err, "getting computer name for new host")
return 0, ctxerr.Wrap(ctx, err, "getting computer name for new host")
}
host := &fleet.AndroidHost{
@@ -959,11 +1131,11 @@ func (svc *Service) addNewHost(ctx context.Context, device *androidmanagement.De
if device.AppliedPolicyName != "" {
policy, err := svc.getPolicyID(ctx, device)
if err != nil {
return ctxerr.Wrap(ctx, err, "getting Android policy ID")
return 0, ctxerr.Wrap(ctx, err, "getting Android policy ID")
}
policySyncTime, err := time.Parse(time.RFC3339, device.LastPolicySyncTime)
if err != nil {
return ctxerr.Wrap(ctx, err, "parsing Android policy sync time")
return 0, ctxerr.Wrap(ctx, err, "parsing Android policy sync time")
}
host.Device.AppliedPolicyID = policy
if device.AppliedPolicyVersion != 0 {
@@ -975,24 +1147,24 @@ func (svc *Service) addNewHost(ctx context.Context, device *androidmanagement.De
fleetHost, err := svc.ds.NewAndroidHost(ctx, host, companyOwned)
if err != nil {
return ctxerr.Wrap(ctx, err, "enrolling Android host")
return 0, ctxerr.Wrap(ctx, err, "enrolling Android host")
}
// Populate the operating_systems table so the host can be filtered via
// `GET /api/v1/fleet/hosts?os_name=Android&os_version=<version>` and show
// up in the /os_versions aggregation alongside other platforms.
if err := svc.updateHostOperatingSystem(ctx, fleetHost.Host.ID, device); err != nil {
return err
return 0, err
}
if enrollmentTokenRequest.IdpUUID != "" {
svc.logger.InfoContext(ctx, "associating android host with idp account", "host_uuid", host.UUID, "idp_uuid", enrollmentTokenRequest.IdpUUID)
err := svc.ds.AssociateHostMDMIdPAccount(ctx, host.UUID, enrollmentTokenRequest.IdpUUID)
if err != nil {
return ctxerr.Wrap(ctx, err, "associating host with idp account")
return 0, ctxerr.Wrap(ctx, err, "associating host with idp account")
}
if err := svc.fleetDS.MaybeAssociateHostWithScimUser(ctx, fleetHost.Host.ID); err != nil {
return ctxerr.Wrap(ctx, err, "associating android host with scim user")
return 0, ctxerr.Wrap(ctx, err, "associating android host with scim user")
}
}
@@ -1004,21 +1176,21 @@ func (svc *Service) addNewHost(ctx context.Context, device *androidmanagement.De
}
if _, err := svc.fleetDS.CreatePendingCertificateTemplatesForNewHost(ctx, fleetHost.Host.UUID, certTeamID); err != nil {
svc.logger.ErrorContext(ctx, "failed to create pending certificate templates for new host", "host_uuid", fleetHost.Host.UUID, "err", err)
return ctxerr.Wrap(ctx, err, "creating pending certificate templates for new host")
return 0, ctxerr.Wrap(ctx, err, "creating pending certificate templates for new host")
}
enterprise, err := svc.ds.GetEnterprise(ctx)
if err != nil {
return ctxerr.Wrap(ctx, err, "get android enterprise")
return 0, ctxerr.Wrap(ctx, err, "get android enterprise")
}
err = worker.QueueRunAndroidSetupExperience(ctx, svc.fleetDS, svc.logger,
fleetHost.Host.UUID, fleetHost.Host.TeamID, enterprise.Name())
if err != nil {
return ctxerr.Wrap(ctx, err, "enqueuing run android setup experience for host job")
return 0, ctxerr.Wrap(ctx, err, "enqueuing run android setup experience for host job")
}
return nil
return fleetHost.Host.ID, nil
}
func getHardwareModel(device *androidmanagement.Device) string {
@@ -0,0 +1,385 @@
package service
import (
"context"
"encoding/base64"
"testing"
"time"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/fleetdm/fleet/v4/server/mdm/android"
common_mysql "github.com/fleetdm/fleet/v4/server/platform/mysql"
"github.com/go-json-experiment/json"
"github.com/stretchr/testify/require"
"google.golang.org/api/androidmanagement/v1"
)
const dedupToken = "value"
// wireDedupHost configures mockDS so both the ENROLLMENT (re-enroll) and
// STATUS_REPORT full-processing paths succeed for a single existing host, and
// returns that host. Individual tests override GetAndroidPubSubDedupStateFunc and
// the invocation flags they assert on.
func wireDedupHost(t *testing.T, mockDS *AndroidMockDS, hostID uint, hostUUID string) *fleet.AndroidHost {
t.Helper()
host := &fleet.AndroidHost{
Host: &fleet.Host{ID: hostID, UUID: hostUUID},
Device: &android.Device{
HostID: hostID,
DeviceID: "existing-device",
EnterpriseSpecificID: &hostUUID,
},
}
mockDS.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
return &fleet.AppConfig{MDM: fleet.MDM{AndroidEnabledAndConfigured: true}}, nil
}
mockDS.AndroidHostLiteFunc = func(ctx context.Context, esID string) (*fleet.AndroidHost, error) {
return host, nil
}
mockDS.UpdateAndroidHostFunc = func(ctx context.Context, h *fleet.AndroidHost, fromEnroll, companyOwned bool) error {
return nil
}
mockDS.VerifyEnrollSecretFunc = func(ctx context.Context, secret string) (*fleet.EnrollSecret, error) {
return &fleet.EnrollSecret{}, nil
}
mockDS.DeleteAllHostCertificateTemplatesFunc = func(ctx context.Context, hostUUID string) error { return nil }
mockDS.ClearHostMDMActionsFunc = func(ctx context.Context, id uint) error { return nil }
mockDS.ScimUserByHostIDFunc = func(ctx context.Context, id uint) (*fleet.ScimUser, error) {
return nil, common_mysql.NotFound("scim user")
}
mockDS.ListHostDeviceMappingFunc = func(ctx context.Context, id uint) ([]*fleet.HostDeviceMapping, error) {
return nil, nil
}
// STATUS_REPORT DELETED path.
mockDS.SetAndroidHostUnenrolledFunc = func(ctx context.Context, id uint) (bool, error) { return true, nil }
mockDS.GetHostMDMFunc = func(ctx context.Context, id uint) (*fleet.HostMDM, error) {
return &fleet.HostMDM{IsPersonalEnrollment: true}, nil
}
mockDS.MarkAllPendingVPPInstallsAsFailedForAndroidHostFunc = func(ctx context.Context, id uint) ([]*fleet.User, []fleet.ActivityDetails, error) {
return nil, nil, nil
}
mockDS.ListHostsLiteByIDsFunc = func(ctx context.Context, ids []uint) ([]*fleet.Host, error) {
return []*fleet.Host{{ID: hostID}}, nil
}
return host
}
// makeEnrollmentEnvelope builds an ENROLLMENT PubSub message for a fixed device
// with the given Google envelope messageId/publishTime.
func makeEnrollmentEnvelope(t *testing.T, messageID, publishTime string) *android.PubSubMessage {
msg := createEnrollmentMessage(t, androidmanagement.Device{
Name: createAndroidDeviceId("dedup"),
EnrollmentTokenData: `{"enroll_secret":"global"}`,
})
msg.MessageID = messageID
msg.PublishTime = publishTime
return msg
}
// makeStatusEnvelope builds a STATUS_REPORT PubSub message (optionally in the
// DELETED state) for a fixed device with the given envelope fields.
func makeStatusEnvelope(t *testing.T, esID, messageID, publishTime string, deleted bool) *android.PubSubMessage {
device := androidmanagement.Device{
Name: createAndroidDeviceId("dedup"),
HardwareInfo: &androidmanagement.HardwareInfo{
EnterpriseSpecificId: esID,
Brand: "TestBrand",
Model: "TestModel",
SerialNumber: "test-serial",
Hardware: "test-hardware",
},
SoftwareInfo: &androidmanagement.SoftwareInfo{AndroidBuildNumber: "test-build", AndroidVersion: "1"},
MemoryInfo: &androidmanagement.MemoryInfo{TotalRam: 8 * 1024 * 1024 * 1024},
}
if deleted {
device.AppliedState = string(android.DeviceStateDeleted)
}
data, err := json.Marshal(device)
require.NoError(t, err)
return &android.PubSubMessage{
Attributes: map[string]string{"notificationType": string(android.PubSubStatusReport)},
Data: base64.StdEncoding.EncodeToString(data),
MessageID: messageID,
PublishTime: publishTime,
}
}
func TestPubSubDedupAndStaleness(t *testing.T) {
const hostID = uint(10)
const hostUUID = "DEDUP-HOST-UUID"
t.Run("duplicate ENROLLMENT messageId is a no-op", func(t *testing.T) {
svc, mockDS := createAndroidService(t)
wireDedupHost(t, mockDS, hostID, hostUUID)
mockDS.GetAndroidPubSubDedupStateFunc = func(ctx context.Context, id uint) (string, *time.Time, error) {
return "msg-dup", nil, nil
}
msg := makeEnrollmentEnvelope(t, "msg-dup", "2026-07-22T10:00:00Z")
require.NoError(t, svc.ProcessPubSubPush(t.Context(), dedupToken, msg))
require.False(t, mockDS.UpdateAndroidHostFuncInvoked, "duplicate enrollment must not re-run updateHost")
require.False(t, mockDS.NewJobFuncInvoked, "duplicate enrollment must not re-queue setup experience")
require.False(t, mockDS.SetAndroidPubSubDedupStateFuncInvoked, "no state should be recorded on a skipped message")
})
t.Run("stale ENROLLMENT event time is skipped", func(t *testing.T) {
svc, mockDS := createAndroidService(t)
wireDedupHost(t, mockDS, hostID, hostUUID)
stored := time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)
mockDS.GetAndroidPubSubDedupStateFunc = func(ctx context.Context, id uint) (string, *time.Time, error) {
return "other-msg", &stored, nil
}
// publishTime older than the stored event time -> stale.
msg := makeEnrollmentEnvelope(t, "msg-new", "2020-01-01T00:00:00Z")
require.NoError(t, svc.ProcessPubSubPush(t.Context(), dedupToken, msg))
require.False(t, mockDS.UpdateAndroidHostFuncInvoked, "stale enrollment must not re-run updateHost")
require.False(t, mockDS.NewJobFuncInvoked, "stale enrollment must not re-queue setup experience")
})
t.Run("re-enrollment records dedup state", func(t *testing.T) {
svc, mockDS := createAndroidService(t)
wireDedupHost(t, mockDS, hostID, hostUUID)
mockDS.GetAndroidPubSubDedupStateFunc = func(ctx context.Context, id uint) (string, *time.Time, error) {
return "older-msg", nil, nil
}
var recordedID string
var recordedHostID uint
mockDS.SetAndroidPubSubDedupStateFunc = func(ctx context.Context, id uint, messageID string, eventTime *time.Time) error {
recordedHostID = id
recordedID = messageID
return nil
}
msg := makeEnrollmentEnvelope(t, "msg-reenroll", "2026-07-22T10:00:00Z")
require.NoError(t, svc.ProcessPubSubPush(t.Context(), dedupToken, msg))
require.True(t, mockDS.UpdateAndroidHostFuncInvoked, "a non-duplicate re-enrollment must run updateHost")
require.True(t, mockDS.SetAndroidPubSubDedupStateFuncInvoked, "successful enrollment must record dedup state")
require.Equal(t, "msg-reenroll", recordedID)
require.Equal(t, hostID, recordedHostID)
})
t.Run("brand-new ENROLLMENT records dedup state without re-reading the host", func(t *testing.T) {
// enrollHost returns the host ID it resolved, so recording dedup state no longer
// depends on a post-enrollment lookup succeeding. A lookup that failed there would
// leave the delivery acked with no dedup state, and the redelivery would re-queue
// the setup experience. AndroidHostLite stays not-found for the whole call to prove
// no such lookup happens after enrollHost.
const newHostID = uint(77)
svc, mockDS := createAndroidService(t)
mockDS.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
return &fleet.AppConfig{MDM: fleet.MDM{AndroidEnabledAndConfigured: true}}, nil
}
var hostLiteCalls int
mockDS.AndroidHostLiteFunc = func(ctx context.Context, esID string) (*fleet.AndroidHost, error) {
hostLiteCalls++
return nil, common_mysql.NotFound("android host lite")
}
mockDS.VerifyEnrollSecretFunc = func(ctx context.Context, secret string) (*fleet.EnrollSecret, error) {
return &fleet.EnrollSecret{}, nil
}
mockDS.NewAndroidHostFunc = func(ctx context.Context, h *fleet.AndroidHost, companyOwned bool) (*fleet.AndroidHost, error) {
return &fleet.AndroidHost{Host: &fleet.Host{ID: newHostID, UUID: hostUUID}, Device: h.Device}, nil
}
var recordedHostID uint
var recordedID string
mockDS.SetAndroidPubSubDedupStateFunc = func(ctx context.Context, id uint, messageID string, eventTime *time.Time) error {
recordedHostID = id
recordedID = messageID
return nil
}
msg := makeEnrollmentEnvelope(t, "msg-brand-new", "2026-07-22T10:00:00Z")
require.NoError(t, svc.ProcessPubSubPush(t.Context(), dedupToken, msg))
require.True(t, mockDS.NewAndroidHostFuncInvoked, "a brand-new device must be inserted")
require.False(t, mockDS.GetAndroidPubSubDedupStateFuncInvoked, "a device new to Fleet has no dedup state to check")
require.True(t, mockDS.SetAndroidPubSubDedupStateFuncInvoked, "new enrollment must record dedup state")
require.Equal(t, newHostID, recordedHostID, "dedup state must be recorded against the newly inserted host")
require.Equal(t, "msg-brand-new", recordedID)
// One lookup for the dedup pre-check, one inside enrollHost. None afterwards.
require.Equal(t, 2, hostLiteCalls, "dedup state must not require a post-enrollment host lookup")
})
t.Run("ENROLLMENT DELETED on an already-unenrolled host emits no activity", func(t *testing.T) {
// The STATUS_REPORT DELETED branch already skips the activity when the state flip
// was a no-op; the ENROLLMENT DELETED branch must match, or a DELETED that arrives
// under both notification types adds a duplicate mdm_unenrolled row.
svc, mockDS := createAndroidService(t)
wireDedupHost(t, mockDS, hostID, hostUUID)
mockDS.GetAndroidPubSubDedupStateFunc = func(ctx context.Context, id uint) (string, *time.Time, error) {
return "", nil, nil // not a duplicate, not stale
}
mockDS.SetAndroidHostUnenrolledFunc = func(ctx context.Context, id uint) (bool, error) {
return false, nil // already unenrolled by an earlier delivery
}
mockDS.SetAndroidPubSubDedupStateFunc = func(ctx context.Context, id uint, messageID string, eventTime *time.Time) error {
return nil
}
msg := makeEnrollmentEnvelope(t, "msg-deleted-enrollment", "2026-07-22T10:00:00Z")
device := androidmanagement.Device{
Name: createAndroidDeviceId("dedup"),
EnrollmentTokenData: `{"enroll_secret":"global"}`,
AppliedState: string(android.DeviceStateDeleted),
HardwareInfo: &androidmanagement.HardwareInfo{
EnterpriseSpecificId: hostUUID,
Brand: "TestBrand",
Model: "TestModel",
},
SoftwareInfo: &androidmanagement.SoftwareInfo{AndroidBuildNumber: "test-build", AndroidVersion: "1"},
MemoryInfo: &androidmanagement.MemoryInfo{TotalRam: 1024},
}
data, err := json.Marshal(device)
require.NoError(t, err)
msg.Data = base64.StdEncoding.EncodeToString(data)
require.NoError(t, svc.ProcessPubSubPush(t.Context(), dedupToken, msg))
require.True(t, mockDS.SetAndroidHostUnenrolledFuncInvoked, "the unenroll must still be attempted")
require.True(t, mockDS.SetAndroidPubSubDedupStateFuncInvoked, "dedup state must still be recorded")
require.False(t, mockDS.ListHostsLiteByIDsFuncInvoked,
"no display-name lookup means no duplicate mdm_unenrolled activity was emitted")
})
t.Run("duplicate STATUS_REPORT messageId is a no-op", func(t *testing.T) {
svc, mockDS := createAndroidService(t)
wireDedupHost(t, mockDS, hostID, hostUUID)
mockDS.GetAndroidPubSubDedupStateFunc = func(ctx context.Context, id uint) (string, *time.Time, error) {
return "msg-dup", nil, nil
}
msg := makeStatusEnvelope(t, hostUUID, "msg-dup", "2026-07-22T10:00:00Z", false)
require.NoError(t, svc.ProcessPubSubPush(t.Context(), dedupToken, msg))
require.False(t, mockDS.UpdateAndroidHostFuncInvoked, "duplicate status report must not re-run updateHost")
require.False(t, mockDS.SetAndroidHostEnrolledFuncInvoked, "duplicate status report must not touch enrollment")
})
t.Run("out-of-order DELETED is skipped by staleness", func(t *testing.T) {
svc, mockDS := createAndroidService(t)
wireDedupHost(t, mockDS, hostID, hostUUID)
// A more-recent re-enrollment was already processed.
stored := time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)
mockDS.GetAndroidPubSubDedupStateFunc = func(ctx context.Context, id uint) (string, *time.Time, error) {
return "recent-enroll-msg", &stored, nil
}
// A stale DELETED redelivered out of order (older publishTime).
msg := makeStatusEnvelope(t, hostUUID, "stale-delete-msg", "2020-01-01T00:00:00Z", true)
require.NoError(t, svc.ProcessPubSubPush(t.Context(), dedupToken, msg))
require.False(t, mockDS.SetAndroidHostUnenrolledFuncInvoked, "a stale DELETED must not unenroll a live host")
})
t.Run("STATUS_REPORT recovers a wrongly-unenrolled host", func(t *testing.T) {
svc, mockDS := createAndroidService(t)
wireDedupHost(t, mockDS, hostID, hostUUID)
mockDS.GetAndroidPubSubDedupStateFunc = func(ctx context.Context, id uint) (string, *time.Time, error) {
return "", nil, nil // no prior state -> processed normally
}
var enrolledHostID uint
mockDS.SetAndroidHostEnrolledFunc = func(ctx context.Context, id uint) (bool, error) {
enrolledHostID = id
return true, nil
}
msg := makeStatusEnvelope(t, hostUUID, "live-msg", "2026-07-22T10:00:00Z", false)
require.NoError(t, svc.ProcessPubSubPush(t.Context(), dedupToken, msg))
require.True(t, mockDS.UpdateAndroidHostFuncInvoked, "a live status report must run updateHost")
require.True(t, mockDS.SetAndroidHostEnrolledFuncInvoked, "a live status report must attempt enrollment recovery")
require.Equal(t, hostID, enrolledHostID)
require.True(t, mockDS.SetAndroidPubSubDedupStateFuncInvoked, "successful status report must record dedup state")
})
t.Run("stale STATUS_REPORT does not trigger enrollment recovery", func(t *testing.T) {
// Mirror of the bug being fixed: a stale STATUS_REPORT (older than the last
// processed event, e.g. one published before a legitimate unenroll) must not
// re-enroll the host. Staleness must short-circuit before SetAndroidHostEnrolled.
svc, mockDS := createAndroidService(t)
wireDedupHost(t, mockDS, hostID, hostUUID)
stored := time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)
mockDS.GetAndroidPubSubDedupStateFunc = func(ctx context.Context, id uint) (string, *time.Time, error) {
return "unenroll-msg", &stored, nil
}
msg := makeStatusEnvelope(t, hostUUID, "stale-report-msg", "2020-01-01T00:00:00Z", false)
require.NoError(t, svc.ProcessPubSubPush(t.Context(), dedupToken, msg))
require.False(t, mockDS.UpdateAndroidHostFuncInvoked, "stale status report must not run updateHost")
require.False(t, mockDS.SetAndroidHostEnrolledFuncInvoked, "stale status report must not re-enroll the host")
})
t.Run("equal event time with a different messageId is processed", func(t *testing.T) {
// A distinct message with the same timestamp is not a duplicate and not stale
// (staleness is strict "older than"), so it must be processed.
svc, mockDS := createAndroidService(t)
wireDedupHost(t, mockDS, hostID, hostUUID)
sameTime := time.Date(2026, 7, 22, 10, 0, 0, 0, time.UTC)
mockDS.GetAndroidPubSubDedupStateFunc = func(ctx context.Context, id uint) (string, *time.Time, error) {
return "stored-msg", &sameTime, nil
}
msg := makeStatusEnvelope(t, hostUUID, "different-msg", "2026-07-22T10:00:00Z", false)
require.NoError(t, svc.ProcessPubSubPush(t.Context(), dedupToken, msg))
require.True(t, mockDS.UpdateAndroidHostFuncInvoked, "a distinct, non-stale message must be processed")
})
t.Run("WIPE ack records dedup state to block a later stale STATUS_REPORT", func(t *testing.T) {
svc, mockDS := createAndroidService(t)
mockDS.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
return &fleet.AppConfig{MDM: fleet.MDM{AndroidEnabledAndConfigured: true}}, nil
}
stored := &android.MDMAndroidCommand{
CommandUUID: "cmd-wipe",
HostUUID: hostUUID,
OperationName: "enterprises/E/devices/D/operations/wipe-ack",
CommandType: string(android.MDMAndroidCommandTypeWipe),
Status: string(android.MDMAndroidCommandStatusPending),
}
mockDS.GetMDMAndroidCommandByOperationNameFunc = func(ctx context.Context, opName string) (*android.MDMAndroidCommand, error) {
return stored, nil
}
mockDS.UpdateMDMAndroidCommandStatusFunc = func(ctx context.Context, commandUUID, status string, errorCode, errorMessage *string) error {
return nil
}
mockDS.AndroidHostLiteByHostUUIDFunc = func(ctx context.Context, hUUID string) (*fleet.AndroidHost, error) {
return &fleet.AndroidHost{Host: &fleet.Host{ID: hostID, UUID: hUUID}}, nil
}
mockDS.GetHostMDMFunc = func(ctx context.Context, id uint) (*fleet.HostMDM, error) {
return &fleet.HostMDM{IsPersonalEnrollment: false}, nil
}
mockDS.ClearHostMDMActionsFunc = func(ctx context.Context, id uint) error { return nil }
mockDS.SetAndroidHostUnenrolledFunc = func(ctx context.Context, id uint) (bool, error) { return true, nil }
mockDS.ListHostsLiteByIDsFunc = func(ctx context.Context, ids []uint) ([]*fleet.Host, error) {
return []*fleet.Host{{ID: hostID}}, nil
}
var recordedID string
var recordedTime *time.Time
mockDS.SetAndroidPubSubDedupStateFunc = func(ctx context.Context, id uint, messageID string, eventTime *time.Time) error {
recordedID = messageID
recordedTime = eventTime
return nil
}
body, err := json.Marshal(androidmanagement.Operation{Name: stored.OperationName, Done: true})
require.NoError(t, err)
msg := &android.PubSubMessage{
Attributes: map[string]string{"notificationType": string(android.PubSubCommand)},
Data: base64.StdEncoding.EncodeToString(body),
MessageID: "wipe-msg",
PublishTime: "2026-07-22T12:00:00Z",
}
require.NoError(t, svc.ProcessPubSubPush(t.Context(), dedupToken, msg))
require.True(t, mockDS.SetAndroidPubSubDedupStateFuncInvoked, "WIPE ack unenroll must record dedup state")
require.Equal(t, "wipe-msg", recordedID)
require.NotNil(t, recordedTime, "WIPE ack must record the notification publish time as the event time")
require.Equal(t, time.Date(2026, 7, 22, 12, 0, 0, 0, time.UTC), recordedTime.UTC())
})
}
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log/slog"
"time"
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
"github.com/fleetdm/fleet/v4/server/fleet"
@@ -88,6 +89,16 @@ func ReconcileAndroidDevices(ctx context.Context, ds fleet.Datastore, logger *sl
logger.ErrorContext(ctx, "failed to mark android host unenrolled during reconcile", "host_id", dev.HostID, "err", derr)
continue
}
// Advance the dedup event time so a STATUS_REPORT that was already in the Pub/Sub
// queue before the AMAPI deletion, delivered afterwards, is dropped as stale by
// handlePubSubStatusReport instead of reverting this unenroll (SetAndroidHostEnrolled
// would otherwise re-enroll the host, causing a flip-flop and a duplicate
// mdm_unenrolled activity on the next reconcile). Best-effort: a missed record only
// weakens dedup for the redelivery window, so log and continue.
now := time.Now().UTC()
if derr := ds.SetAndroidPubSubDedupState(ctx, dev.HostID, "", &now); derr != nil {
logger.WarnContext(ctx, "failed to record android pubsub dedup state during reconcile", "host_id", dev.HostID, "err", derr)
}
// Emit system activity to mirror Pub/Sub DELETED handling.
var displayName, serial string
if hosts, herr := ds.ListHostsLiteByIDs(ctx, []uint{dev.HostID}); herr == nil && len(hosts) == 1 && hosts[0] != nil {
+36
View File
@@ -1952,6 +1952,12 @@ type BulkSetAndroidHostsUnenrolledFunc func(ctx context.Context) error
type SetAndroidHostUnenrolledFunc func(ctx context.Context, hostID uint) (bool, error)
type SetAndroidHostEnrolledFunc func(ctx context.Context, hostID uint) (bool, error)
type GetAndroidPubSubDedupStateFunc func(ctx context.Context, hostID uint) (messageID string, eventTime *time.Time, err error)
type SetAndroidPubSubDedupStateFunc func(ctx context.Context, hostID uint, messageID string, eventTime *time.Time) error
type NewAndroidHostFunc func(ctx context.Context, host *fleet.AndroidHost, companyOwned bool) (*fleet.AndroidHost, error)
type SetAndroidEnabledAndConfiguredFunc func(ctx context.Context, configured bool) error
@@ -5183,6 +5189,15 @@ type DataStore struct {
SetAndroidHostUnenrolledFunc SetAndroidHostUnenrolledFunc
SetAndroidHostUnenrolledFuncInvoked bool
SetAndroidHostEnrolledFunc SetAndroidHostEnrolledFunc
SetAndroidHostEnrolledFuncInvoked bool
GetAndroidPubSubDedupStateFunc GetAndroidPubSubDedupStateFunc
GetAndroidPubSubDedupStateFuncInvoked bool
SetAndroidPubSubDedupStateFunc SetAndroidPubSubDedupStateFunc
SetAndroidPubSubDedupStateFuncInvoked bool
NewAndroidHostFunc NewAndroidHostFunc
NewAndroidHostFuncInvoked bool
@@ -12441,6 +12456,27 @@ func (s *DataStore) SetAndroidHostUnenrolled(ctx context.Context, hostID uint) (
return s.SetAndroidHostUnenrolledFunc(ctx, hostID)
}
func (s *DataStore) SetAndroidHostEnrolled(ctx context.Context, hostID uint) (bool, error) {
s.mu.Lock()
s.SetAndroidHostEnrolledFuncInvoked = true
s.mu.Unlock()
return s.SetAndroidHostEnrolledFunc(ctx, hostID)
}
func (s *DataStore) GetAndroidPubSubDedupState(ctx context.Context, hostID uint) (messageID string, eventTime *time.Time, err error) {
s.mu.Lock()
s.GetAndroidPubSubDedupStateFuncInvoked = true
s.mu.Unlock()
return s.GetAndroidPubSubDedupStateFunc(ctx, hostID)
}
func (s *DataStore) SetAndroidPubSubDedupState(ctx context.Context, hostID uint, messageID string, eventTime *time.Time) error {
s.mu.Lock()
s.SetAndroidPubSubDedupStateFuncInvoked = true
s.mu.Unlock()
return s.SetAndroidPubSubDedupStateFunc(ctx, hostID, messageID, eventTime)
}
func (s *DataStore) NewAndroidHost(ctx context.Context, host *fleet.AndroidHost, companyOwned bool) (*fleet.AndroidHost, error) {
s.mu.Lock()
s.NewAndroidHostFuncInvoked = true