diff --git a/changes/41910-throttle-AMAPI b/changes/41910-throttle-AMAPI new file mode 100644 index 0000000000..5ce3995834 --- /dev/null +++ b/changes/41910-throttle-AMAPI @@ -0,0 +1 @@ +- Added configurable batch size `FLEET_MDM_ANDROID_BATCH_SIZE` (default: 1000 hosts) for Android MDM operations to prevent overwhelming the Google Android Management API. diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index f812a6fd4e..41c444b65b 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -1930,6 +1930,7 @@ func newAndroidMDMProfileManagerSchedule( logger *slog.Logger, licenseKey string, androidAgentConfig config.AndroidAgentConfig, + batchSize int, ) (*schedule.Schedule, error) { const ( name = string(fleet.CronMDMAndroidProfileManager) @@ -1941,7 +1942,7 @@ func newAndroidMDMProfileManagerSchedule( ctx, name, instanceID, defaultInterval, ds, ds, schedule.WithLogger(logger), schedule.WithJob("manage_android_profiles", func(ctx context.Context) error { - return android_svc.ReconcileProfiles(ctx, ds, logger, licenseKey, androidAgentConfig) + return android_svc.ReconcileProfiles(ctx, ds, logger, licenseKey, androidAgentConfig, batchSize) }), ) diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 72566e3b3a..2814e41324 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -443,6 +443,7 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev // Declare svc early so the closure below can capture it. var svc fleet.Service config.MDM.AndroidAgent.Validate(initFatal) + config.MDM.ValidateAndroidBatchSize(initFatal) androidSvc, err := android_service.NewService( ctx, logger, @@ -829,6 +830,7 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev logger, config.License.Key, // NOTE: this requires the license key, not the parsed *LicenseInfo available in the ctx config.MDM.AndroidAgent, + config.MDM.AndroidBatchSize, ) }); err != nil { initFatal(err, "failed to register mdm_android_profile_manager schedule") diff --git a/server/config/config.go b/server/config/config.go index 5ab6021242..8e85f9cba5 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -926,13 +926,22 @@ type MDMConfig struct { EnableCustomFileVault bool `yaml:"enable_custom_filevault"` AllowAllDeclarations bool `yaml:"allow_all_declarations"` - AndroidAgent AndroidAgentConfig `yaml:"android_agent"` + AndroidAgent AndroidAgentConfig `yaml:"android_agent"` + AndroidBatchSize int `yaml:"android_batch_size"` } func (m MDMConfig) IsCustomFileVaultEnabled() bool { return m.EnableCustomOSUpdatesAndFileVault || m.EnableCustomFileVault } +// ValidateAndroidBatchSize checks that the configured batch size is non-negative. +func (m MDMConfig) ValidateAndroidBatchSize(initFatal func(err error, msg string)) { + if m.AndroidBatchSize < 0 { + initFatal(errors.New("mdm.android_batch_size must be non-negative (0 = no limit)"), + "Android MDM configuration") + } +} + // AndroidAgentConfig holds configuration for the Fleet Android agent. type AndroidAgentConfig struct { // Package is the package name for the Fleet Android agent. @@ -1758,6 +1767,8 @@ func (man Manager) addConfigs() { man.addConfigString("mdm.android_agent.signing_sha256", "x+IyvrwVbQEBYV/ojWmLavJE0VIZE1RAT2JmxeI5sFw=", "Signing certificate SHA256 fingerprint for the Fleet Android agent") man.hideConfig("mdm.android_agent.package") man.hideConfig("mdm.android_agent.signing_sha256") + man.addConfigInt("mdm.android_batch_size", 1000, "Maximum number of hosts per batch for Android MDM API operations (1000 default; 0 = no limit)") + man.hideConfig("mdm.android_batch_size") // Calendar integration man.addConfigDuration( @@ -2091,6 +2102,7 @@ func (man Manager) LoadConfig() FleetConfig { Package: man.getConfigString("mdm.android_agent.package"), SigningSHA256: man.getConfigString("mdm.android_agent.signing_sha256"), }, + AndroidBatchSize: man.getConfigInt("mdm.android_batch_size"), }, Calendar: CalendarConfig{ Periodicity: man.getConfigDuration("calendar.periodicity"), diff --git a/server/config/config_test.go b/server/config/config_test.go index 15115e29d7..d5d8d97d16 100644 --- a/server/config/config_test.go +++ b/server/config/config_test.go @@ -803,6 +803,27 @@ func TestAndroidAgentConfigValidate(t *testing.T) { }) } +func TestAndroidBatchSizeValidate(t *testing.T) { + t.Parallel() + + t.Run("valid when positive", func(t *testing.T) { + cfg := MDMConfig{AndroidBatchSize: 1000} + cfg.ValidateAndroidBatchSize(func(err error, msg string) { t.Fatalf("unexpected error: %v", err) }) + }) + + t.Run("valid when zero", func(t *testing.T) { + cfg := MDMConfig{AndroidBatchSize: 0} + cfg.ValidateAndroidBatchSize(func(err error, msg string) { t.Fatalf("unexpected error: %v", err) }) + }) + + t.Run("invalid when negative", func(t *testing.T) { + cfg := MDMConfig{AndroidBatchSize: -1} + called := false + cfg.ValidateAndroidBatchSize(func(err error, msg string) { called = true }) + require.True(t, called) + }) +} + func TestServerConfigWithH2C(t *testing.T) { ctx := context.Background() diff --git a/server/datastore/mysql/android.go b/server/datastore/mysql/android.go index ab82284adf..26658dbcd3 100644 --- a/server/datastore/mysql/android.go +++ b/server/datastore/mysql/android.go @@ -1313,6 +1313,18 @@ const androidApplicableProfilesQuery = ` count_host_updated_after_labels = 0 ` +// GetMDMAndroidReconcileCursor is a no-op on the bare mysql.Datastore; +// the mysqlredis wrapper backs it with Redis. +func (ds *Datastore) GetMDMAndroidReconcileCursor(_ context.Context) (string, error) { + return "", nil +} + +// SetMDMAndroidReconcileCursor is a no-op on the bare mysql.Datastore; +// the mysqlredis wrapper writes to Redis. +func (ds *Datastore) SetMDMAndroidReconcileCursor(_ context.Context, _ string) error { + return nil +} + // ListMDMAndroidProfilesToSend is the android platform equivalent to // ListMDMAppleProfilesToInstall/Remove and // ListMDMWindowsProfilesToInstall/Remove. It plays a similar role but is quite @@ -1336,12 +1348,28 @@ const androidApplicableProfilesQuery = ` // // See https://github.com/fleetdm/fleet/issues/32032#issuecomment-3229548389 // for more details on the rationale of that approach. -func (ds *Datastore) ListMDMAndroidProfilesToSend(ctx context.Context) ([]*fleet.MDMAndroidProfilePayload, []*fleet.MDMAndroidProfilePayload, error) { +func (ds *Datastore) ListMDMAndroidProfilesToSend(ctx context.Context, cursor string, batchSize int) ([]*fleet.MDMAndroidProfilePayload, []*fleet.MDMAndroidProfilePayload, error) { var toApplyProfiles, toRemoveProfiles []*fleet.MDMAndroidProfilePayload err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { + var installCursorPred, removeCursorPred string + var args []any + if cursor != "" { + installCursorPred = "AND ds.host_uuid > ?" + removeCursorPred = "AND hmap.host_uuid > ?" + args = []any{cursor, fleet.MDMOperationTypeRemove, fleet.MDMDeliveryPending, cursor} + } else { + args = []any{fleet.MDMOperationTypeRemove, fleet.MDMDeliveryPending} + } + + var limitClause string + if batchSize > 0 { + limitClause = fmt.Sprintf("LIMIT %d", batchSize) + } + hostsWithChangesStmt := fmt.Sprintf(` WITH ds AS ( %s ) + SELECT host_uuid FROM ( SELECT DISTINCT ds.host_uuid FROM ds @@ -1362,6 +1390,7 @@ func (ds *Datastore) ListMDMAndroidProfilesToSend(ctx context.Context) ([]*fleet -- profile needs retry (status reset to NULL after transient failure) hmap.status IS NULL ) + %s UNION @@ -1382,7 +1411,15 @@ func (ds *Datastore) ListMDMAndroidProfilesToSend(ctx context.Context) ([]*fleet ds.host_uuid IS NULL AND -- and it is not in pending remove status (in which case it was processed) ( hmap.operation_type != ? OR COALESCE(hmap.status, '') <> ? ) -`, fmt.Sprintf(androidApplicableProfilesQuery, "TRUE", "TRUE", "TRUE", "TRUE", "TRUE", "TRUE")) + %s + ) AS all_changes + ORDER BY host_uuid + %s +`, fmt.Sprintf(androidApplicableProfilesQuery, "TRUE", "TRUE", "TRUE", "TRUE", "TRUE", "TRUE"), + installCursorPred, + removeCursorPred, + limitClause, + ) // NOTE: we explicitly don't "ignore" profiles to remove based on broken labels, // because of how Android profiles are applied vs other platforms (ignoring @@ -1397,8 +1434,7 @@ func (ds *Datastore) ListMDMAndroidProfilesToSend(ctx context.Context) ([]*fleet // see https://github.com/fleetdm/fleet/issues/25557#issuecomment-3246496873 var hostUUIDs []string - if err := sqlx.SelectContext(ctx, tx, &hostUUIDs, hostsWithChangesStmt, - fleet.MDMOperationTypeRemove, fleet.MDMDeliveryPending); err != nil { + if err := sqlx.SelectContext(ctx, tx, &hostUUIDs, hostsWithChangesStmt, args...); err != nil { return ctxerr.Wrap(ctx, err, "list android hosts with profile changes") } @@ -1886,7 +1922,7 @@ func (ds *Datastore) bulkSetPendingMDMAndroidHostProfilesDB( return false, nil } - profilesToInstall, profilesToRemove, err := ds.ListMDMAndroidProfilesToSend(ctx) + profilesToInstall, profilesToRemove, err := ds.ListMDMAndroidProfilesToSend(ctx, "", 0) if err != nil { return false, ctxerr.Wrap(ctx, err, "list android profiles to send") } diff --git a/server/datastore/mysql/android_test.go b/server/datastore/mysql/android_test.go index 12aac25729..b1db058081 100644 --- a/server/datastore/mysql/android_test.go +++ b/server/datastore/mysql/android_test.go @@ -1,9 +1,12 @@ package mysql import ( + "cmp" "context" "encoding/json" "fmt" + "maps" + "slices" "strings" "testing" "time" @@ -39,6 +42,7 @@ func TestAndroid(t *testing.T) { {"ListMDMAndroidProfilesToSend", testListMDMAndroidProfilesToSend}, {"ListMDMAndroidProfilesToSend_WithExcludeAny", testListMDMAndroidProfilesToSendWithExcludeAny}, {"ListMDMAndroidProfilesToSend_WithCombinedLabels", testListMDMAndroidProfilesToSendWithCombinedLabels}, + {"ListMDMAndroidProfilesToSend_Cursor", testListMDMAndroidProfilesToSendCursor}, {"GetMDMAndroidProfilesContents", testGetMDMAndroidProfilesContents}, {"BulkUpsertMDMAndroidHostProfiles", testBulkUpsertMDMAndroidHostProfiles}, {"BulkUpsertMDMAndroidHostProfiles", testBulkUpsertMDMAndroidHostProfiles2}, @@ -1347,7 +1351,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { } // without any profile, should return empty - profs, toRemoveProfs, err := ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err := ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, profs) require.Empty(t, toRemoveProfs) @@ -1369,7 +1373,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { profChecksum := getAndroidProfileChecksum(t, ds, p1.ProfileUUID) // both no-team profiles should be applicable to both hosts - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 4) @@ -1385,7 +1389,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { require.NoError(t, err) // profiles for host 1 change to p3 - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 3) @@ -1404,7 +1408,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { require.NoError(t, err) // no change, host is not a member of both labels - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 3) @@ -1419,7 +1423,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { require.NoError(t, err) // no change, host is not a member of both labels - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 3) @@ -1434,7 +1438,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { require.NoError(t, err) // now p4 is applicable to host 0 - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 4) @@ -1454,7 +1458,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { require.NoError(t, err) // no change, host 0 not a member yet - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 4) @@ -1470,7 +1474,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { require.NoError(t, err) // now p5 is applicable to host 0 - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 5) @@ -1491,7 +1495,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { require.NoError(t, err) // no change, label membership was not updated after labels created - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 5) @@ -1510,7 +1514,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { require.NoError(t, err) // host 0 is _not_ a member of the excluded labels, so p6 is applicable - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 6) @@ -1528,7 +1532,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { require.NoError(t, err) // p6 is not applicable anymore - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 5) @@ -1549,7 +1553,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { require.NoError(t, err) // it is not included in noProfHosts as it has p3 - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 6) @@ -1571,7 +1575,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { }) // host 2 is not included in the results as it has p3 installed - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 5) @@ -1588,7 +1592,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { require.NoError(t, err) // host 2 is now a host with no profile (profile 3 needs to be cleared), host 1 is unlisted as it didn't have p3 installed - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.ElementsMatch(t, []*fleet.MDMAndroidProfilePayload{ {ProfileUUID: p3.ProfileUUID, HostUUID: hosts[2].UUID, ProfileName: p3.Name, Checksum: profChecksum}, @@ -1607,7 +1611,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { return err }) - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 4) @@ -1623,7 +1627,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) { _, err := q.ExecContext(ctx, `UPDATE host_mdm SET enrolled=0 WHERE host_id=?`, hosts[0].ID) return err }) - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, profs) require.Empty(t, toRemoveProfs) @@ -1649,7 +1653,7 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore) } // without any profile, should return empty - profs, toRemoveProfs, err := ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err := ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, profs) require.Empty(t, toRemoveProfs) @@ -1682,7 +1686,7 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore) profChecksum := getAndroidProfileChecksum(t, ds, p1.ProfileUUID) // p2 becomes immediately applicable because it only has a manual label - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 1) @@ -1697,7 +1701,7 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore) require.NoError(t, err) // host 0 dynamic labels now apply, and this host is _not_ a member of the excluded labels, so p1, p2 and p3 are now applicable - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 3) @@ -1727,7 +1731,7 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore) require.NoError(t, err) // p5 becomes immediately applicable to host 1 because it only has a manual label - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 4) @@ -1746,7 +1750,7 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore) require.NoError(t, ds.AddHostsToTeam(ctx, fleet.NewAddHostsToTeamParams(&tm.ID, []uint{hosts[1].ID}))) hosts[1].TeamID = &tm.ID - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 6) @@ -1763,7 +1767,7 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore) _, _, err = ds.UpdateLabelMembershipByHostIDs(ctx, *lblExclAny2, []uint{hosts[0].ID}, fleet.TeamFilter{}) require.NoError(t, err) - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 4) @@ -1780,7 +1784,7 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore) _, _, err = ds.UpdateLabelMembershipByHostIDs(ctx, *lblExclAny1, []uint{hosts[0].ID, hosts[1].ID}, fleet.TeamFilter{}) require.NoError(t, err) - profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.Len(t, profs, 1) @@ -1789,6 +1793,89 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore) }, profs) } +func testListMDMAndroidProfilesToSendCursor(t *testing.T, ds *Datastore) { + test.AddBuiltinLabels(t, ds) + ctx := t.Context() + + // Create 5 hosts with predictable UUIDs for cursor ordering. + hosts := make([]*fleet.Host, 5) + for i := range hosts { + androidHost := createAndroidHost(fmt.Sprintf("cursor-host-%02d", i)) + newHost, err := ds.NewAndroidHost(ctx, androidHost, false) + require.NoError(t, err) + hosts[i] = newHost.Host + } + + // Sort by UUID so we can predict cursor order. + slices.SortFunc(hosts, func(a, b *fleet.Host) int { + return cmp.Compare(a.UUID, b.UUID) + }) + + // Add a profile so all 5 hosts have pending work. + _, err := ds.NewMDMAndroidConfigProfile(ctx, *androidProfileForTest("cursor-test-profile")) + require.NoError(t, err) + + // No cursor, no limit — returns all 5 hosts. + allProfs, _, err := ds.ListMDMAndroidProfilesToSend(ctx, "", 0) + require.NoError(t, err) + allHostUUIDs := make(map[string]struct{}) + for _, p := range allProfs { + allHostUUIDs[p.HostUUID] = struct{}{} + } + require.Len(t, allHostUUIDs, 5) + + // Batch 1: limit 2 hosts, no cursor. + batch1Profs, _, err := ds.ListMDMAndroidProfilesToSend(ctx, "", 2) + require.NoError(t, err) + batch1Hosts := make(map[string]struct{}) + for _, p := range batch1Profs { + batch1Hosts[p.HostUUID] = struct{}{} + } + require.Len(t, batch1Hosts, 2, "batch 1 should return exactly 2 hosts") + + // Hosts should be the first 2 in sorted order. + sorted1 := slices.Sorted(maps.Keys(batch1Hosts)) + require.Equal(t, hosts[0].UUID, sorted1[0]) + require.Equal(t, hosts[1].UUID, sorted1[1]) + + // Batch 2: cursor past the last host of batch 1, limit 2. + cursor := sorted1[len(sorted1)-1] + batch2Profs, _, err := ds.ListMDMAndroidProfilesToSend(ctx, cursor, 2) + require.NoError(t, err) + batch2Hosts := make(map[string]struct{}) + for _, p := range batch2Profs { + batch2Hosts[p.HostUUID] = struct{}{} + } + require.Len(t, batch2Hosts, 2, "batch 2 should return exactly 2 hosts") + + // No overlap with batch 1, and all hosts should be > cursor. + sorted2 := slices.Sorted(maps.Keys(batch2Hosts)) + for _, uuid := range sorted2 { + require.Greater(t, uuid, cursor, "batch 2 hosts must be after cursor") + _, overlap := batch1Hosts[uuid] + require.False(t, overlap, "batch 2 must not overlap with batch 1") + } + + // Batch 3: cursor past batch 2, limit 2 — should return the remaining 1 host. + cursor = sorted2[len(sorted2)-1] + batch3Profs, _, err := ds.ListMDMAndroidProfilesToSend(ctx, cursor, 2) + require.NoError(t, err) + batch3Hosts := make(map[string]struct{}) + for _, p := range batch3Profs { + batch3Hosts[p.HostUUID] = struct{}{} + } + require.Len(t, batch3Hosts, 1, "batch 3 should return the remaining 1 host") + + sorted3 := slices.Sorted(maps.Keys(batch3Hosts)) + require.Greater(t, sorted3[0], cursor, "batch 3 host must be after cursor") + + // Batch 4: cursor past batch 3 — should return empty (end of pass). + cursor = sorted3[0] + batch4Profs, _, err := ds.ListMDMAndroidProfilesToSend(ctx, cursor, 2) + require.NoError(t, err) + require.Empty(t, batch4Profs, "no more hosts after end of universe") +} + func testListMDMAndroidProfilesToSendWithCombinedLabels(t *testing.T, ds *Datastore) { test.AddBuiltinLabels(t, ds) ctx := t.Context() @@ -1823,7 +1910,7 @@ func testListMDMAndroidProfilesToSendWithCombinedLabels(t *testing.T, ds *Datast profChecksum := getAndroidProfileChecksum(t, ds, pCombinedAll.ProfileUUID) // host is not a member of any label → neither profile applies - profs, toRemove, err := ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemove, err := ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemove) require.Empty(t, profs) @@ -1832,7 +1919,7 @@ func testListMDMAndroidProfilesToSendWithCombinedLabels(t *testing.T, ds *Datast err = ds.AddLabelsToHost(ctx, h.ID, []uint{inclAllLbl.ID, inclAllLbl2.ID, inclAnyLbl.ID}) require.NoError(t, err) - profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemove) require.Len(t, profs, 2) @@ -1845,7 +1932,7 @@ func testListMDMAndroidProfilesToSendWithCombinedLabels(t *testing.T, ds *Datast err = ds.AddLabelsToHost(ctx, h.ID, []uint{exclLbl.ID}) require.NoError(t, err) - profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemove) require.Empty(t, profs) @@ -1854,7 +1941,7 @@ func testListMDMAndroidProfilesToSendWithCombinedLabels(t *testing.T, ds *Datast err = ds.RemoveLabelsFromHost(ctx, h.ID, []uint{exclLbl.ID}) require.NoError(t, err) - profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemove) require.Len(t, profs, 2) @@ -1867,7 +1954,7 @@ func testListMDMAndroidProfilesToSendWithCombinedLabels(t *testing.T, ds *Datast err = ds.RemoveLabelsFromHost(ctx, h.ID, []uint{inclAllLbl2.ID}) require.NoError(t, err) - profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx) + profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemove) require.Len(t, profs, 1) @@ -1976,7 +2063,7 @@ func testBulkUpsertMDMAndroidHostProfilesN(t *testing.T, ds *Datastore, batchSiz ds.testUpsertMDMDesiredProfilesBatchSize = batchSize t.Cleanup(func() { ds.testUpsertMDMDesiredProfilesBatchSize = 0 }) - hostProfiles, toRemoveProfs, err := ds.ListMDMAndroidProfilesToSend(ctx) + hostProfiles, toRemoveProfs, err := ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.ElementsMatch(t, []*fleet.MDMAndroidProfilePayload{ @@ -2019,7 +2106,7 @@ func testBulkUpsertMDMAndroidHostProfilesN(t *testing.T, ds *Datastore, batchSiz }) require.NoError(t, err) - hostProfiles, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + hostProfiles, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.ElementsMatch(t, []*fleet.MDMAndroidProfilePayload{ @@ -2061,7 +2148,7 @@ func testBulkUpsertMDMAndroidHostProfilesN(t *testing.T, ds *Datastore, batchSiz }) require.NoError(t, err) - hostProfiles, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + hostProfiles, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.Empty(t, toRemoveProfs) require.ElementsMatch(t, []*fleet.MDMAndroidProfilePayload{ @@ -2076,7 +2163,7 @@ func testBulkUpsertMDMAndroidHostProfilesN(t *testing.T, ds *Datastore, batchSiz err = ds.DeleteMDMAndroidConfigProfile(ctx, profiles[2].ProfileUUID) require.NoError(t, err) - hostProfiles, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx) + hostProfiles, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0) require.NoError(t, err) require.ElementsMatch(t, []*fleet.MDMAndroidProfilePayload{ {ProfileUUID: profiles[2].ProfileUUID, HostUUID: hosts[2].UUID, ProfileName: profiles[2].Name, Checksum: profChecksum}, diff --git a/server/datastore/mysqlredis/android_recon_cursor.go b/server/datastore/mysqlredis/android_recon_cursor.go new file mode 100644 index 0000000000..44bc6777fc --- /dev/null +++ b/server/datastore/mysqlredis/android_recon_cursor.go @@ -0,0 +1,49 @@ +package mysqlredis + +import ( + "context" + "errors" + + "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/datastore/redis" + redigo "github.com/gomodule/redigo/redis" +) + +// androidReconCursorKey is the Redis key holding the host_uuid cursor for +// the batched Android MDM profile reconciler. +const androidReconCursorKey = "mdm:android:recon_cursor" + +// GetMDMAndroidReconcileCursor returns the persisted host_uuid cursor used by +// the batched Android MDM reconciliation cron to bound per-tick work. +// +// Returns "" if the key is unset (fresh deployment, Redis flushed, or full +// pass complete). Loss of this key is harmless: the cron resumes from the +// beginning. The desired-state diff is recomputed every tick, so re- +// processing converges naturally. +func (d *Datastore) GetMDMAndroidReconcileCursor(ctx context.Context) (string, error) { + conn := redis.ConfigureDoer(d.pool, d.pool.Get()) + defer conn.Close() + + cursor, err := redigo.String(conn.Do("GET", androidReconCursorKey)) + switch { + case err == nil: + return cursor, nil + case errors.Is(err, redigo.ErrNil): + return "", nil + default: + return "", ctxerr.Wrap(ctx, err, "get android MDM reconcile cursor") + } +} + +// SetMDMAndroidReconcileCursor persists the host_uuid cursor used by the +// batched Android MDM reconciliation cron. An empty string indicates a full +// pass has completed; the next tick will start from the beginning. +func (d *Datastore) SetMDMAndroidReconcileCursor(ctx context.Context, cursor string) error { + conn := redis.ConfigureDoer(d.pool, d.pool.Get()) + defer conn.Close() + + if _, err := conn.Do("SET", androidReconCursorKey, cursor); err != nil { + return ctxerr.Wrap(ctx, err, "set android MDM reconcile cursor") + } + return nil +} diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index e36aa50f5e..54bc7903db 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -3131,7 +3131,18 @@ type Datastore interface { // ListMDMAndroidProfilesToSend lists the Android hosts that need to have // their configuration profiles (Android policy) sent. It returns two lists, // the list of profiles to apply and the list of profiles to remove. - ListMDMAndroidProfilesToSend(ctx context.Context) ([]*MDMAndroidProfilePayload, []*MDMAndroidProfilePayload, error) + // When cursor is non-empty, only hosts with host_uuid > cursor are + // considered. When batchSize > 0, at most batchSize distinct hosts are + // returned. + ListMDMAndroidProfilesToSend(ctx context.Context, cursor string, batchSize int) ([]*MDMAndroidProfilePayload, []*MDMAndroidProfilePayload, error) + + // GetMDMAndroidReconcileCursor returns the persisted host_uuid cursor + // used by the Android MDM reconciliation cron to bound per-tick work. + GetMDMAndroidReconcileCursor(ctx context.Context) (string, error) + + // SetMDMAndroidReconcileCursor persists the host_uuid cursor used by + // the Android MDM reconciliation cron. See GetMDMAndroidReconcileCursor. + SetMDMAndroidReconcileCursor(ctx context.Context, cursor string) error // GetMDMAndroidProfilesContents retrieves the contents of the Android // profiles with the specified UUIDs. diff --git a/server/mdm/android/service/profiles.go b/server/mdm/android/service/profiles.go index b1b5807c56..66a987dfdb 100644 --- a/server/mdm/android/service/profiles.go +++ b/server/mdm/android/service/profiles.go @@ -19,13 +19,13 @@ import ( "google.golang.org/api/androidmanagement/v1" ) -func ReconcileProfiles(ctx context.Context, ds fleet.Datastore, logger *slog.Logger, licenseKey string, androidAgentConfig config.AndroidAgentConfig) error { - return ReconcileProfilesWithClient(ctx, ds, logger, licenseKey, nil, androidAgentConfig) +func ReconcileProfiles(ctx context.Context, ds fleet.Datastore, logger *slog.Logger, licenseKey string, androidAgentConfig config.AndroidAgentConfig, batchSize int) error { + return ReconcileProfilesWithClient(ctx, ds, logger, licenseKey, nil, androidAgentConfig, batchSize) } // ReconcileProfilesWithClient is like ReconcileProfiles but allows injecting a custom client for testing. // If client is nil, a new AMAPI client will be created. -func ReconcileProfilesWithClient(ctx context.Context, ds fleet.Datastore, logger *slog.Logger, licenseKey string, client androidmgmt.Client, androidAgentConfig config.AndroidAgentConfig) error { +func ReconcileProfilesWithClient(ctx context.Context, ds fleet.Datastore, logger *slog.Logger, licenseKey string, client androidmgmt.Client, androidAgentConfig config.AndroidAgentConfig, batchSize int) (err error) { appConfig, err := ds.AppConfig(ctx) if err != nil { return ctxerr.Wrap(ctx, err, "get app config") @@ -54,6 +54,14 @@ func ReconcileProfilesWithClient(ctx context.Context, ds fleet.Datastore, logger } } + // Read the cursor; on error, treat as start-of-pass and continue. + cursor, cursorErr := ds.GetMDMAndroidReconcileCursor(ctx) + if cursorErr != nil { + logger.WarnContext(ctx, "failed to read android MDM reconcile cursor; starting from beginning", + "err", cursorErr) + cursor = "" + } + reconciler := &profileReconciler{ DS: ds, Enterprise: enterprise, @@ -61,7 +69,24 @@ func ReconcileProfilesWithClient(ctx context.Context, ds fleet.Datastore, logger AndroidAgentConfig: androidAgentConfig, Logger: logger, } - return reconciler.ReconcileProfiles(ctx) + hostCount, err := reconciler.ReconcileProfiles(ctx, cursor, batchSize) + + var nextCursor string + if batchSize > 0 && hostCount >= batchSize { + // reconciler returns hostCount matching the number of distinct + // host UUIDs it processed; advance past them. + nextCursor = reconciler.lastHostUUID + } + + // On success, advance the cursor. On failure, leave it where it was + // so the next tick retries the same host window. + if err == nil && cursor != nextCursor { + if cerr := ds.SetMDMAndroidReconcileCursor(ctx, nextCursor); cerr != nil { + logger.WarnContext(ctx, "failed to advance android MDM reconcile cursor", "err", cerr) + } + } + + return err } // profileReconciler is a struct to facilitate testability, it should not be @@ -72,6 +97,7 @@ type profileReconciler struct { Client androidmgmt.Client AndroidAgentConfig config.AndroidAgentConfig Logger *slog.Logger + lastHostUUID string } func getClientAuthenticationSecret(ctx context.Context, ds fleet.Datastore) (string, error) { @@ -85,15 +111,15 @@ func getClientAuthenticationSecret(ctx context.Context, ds fleet.Datastore) (str return string(assets[fleet.MDMAssetAndroidFleetServerSecret].Value), nil } -func (r *profileReconciler) ReconcileProfiles(ctx context.Context) error { +func (r *profileReconciler) ReconcileProfiles(ctx context.Context, cursor string, batchSize int) (int, error) { if err := r.reconcileCertificateTemplates(ctx); err != nil { - return ctxerr.Wrap(ctx, err, "reconcile certificate templates") + return 0, ctxerr.Wrap(ctx, err, "reconcile certificate templates") } // get the list of hosts that need to have their profiles applied - hostsApplicableProfiles, hostsProfsToRemove, err := r.DS.ListMDMAndroidProfilesToSend(ctx) + hostsApplicableProfiles, hostsProfsToRemove, err := r.DS.ListMDMAndroidProfilesToSend(ctx, cursor, batchSize) if err != nil { - return ctxerr.Wrap(ctx, err, "identify android profiles to send") + return 0, ctxerr.Wrap(ctx, err, "identify android profiles to send") } profilesByHostUUID := make(map[string][]*fleet.MDMAndroidProfilePayload) @@ -108,7 +134,7 @@ func (r *profileReconciler) ReconcileProfiles(ctx context.Context) error { profilesContents, err := r.DS.GetMDMAndroidProfilesContents(ctx, slices.Collect(maps.Keys(profilesToLoad))) if err != nil { - return ctxerr.Wrap(ctx, err, "load android profiles content") + return 0, ctxerr.Wrap(ctx, err, "load android profiles content") } // index the to-remove profiles by host so we can pass them to sendHostProfiles @@ -117,6 +143,21 @@ func (r *profileReconciler) ReconcileProfiles(ctx context.Context) error { toRemoveByHostUUID[prof.HostUUID] = append(toRemoveByHostUUID[prof.HostUUID], prof) } + // Collect all distinct host UUIDs for cursor advancement. + allHostUUIDs := make(map[string]struct{}, len(profilesByHostUUID)+len(toRemoveByHostUUID)) + for uuid := range profilesByHostUUID { + allHostUUIDs[uuid] = struct{}{} + } + for uuid := range toRemoveByHostUUID { + allHostUUIDs[uuid] = struct{}{} + } + hostCount := len(allHostUUIDs) + + // Track the last (lexicographically greatest) host UUID for cursor. + if hostCount > 0 { + r.lastHostUUID = slices.Max(slices.Collect(maps.Keys(allHostUUIDs))) + } + // Extract ONC cert aliases once for all hosts (profile contents are shared), // then batch-fetch cert statuses for all hosts in a single DB query. certAliases := extractProfileCertAliases(ctx, r.Logger, profilesContents) @@ -124,7 +165,7 @@ func (r *profileReconciler) ReconcileProfiles(ctx context.Context) error { if len(certAliases) > 0 { allCertStatuses, err = r.DS.GetCertificateTemplateStatusesByNameForHosts(ctx, slices.Collect(maps.Keys(profilesByHostUUID))) if err != nil { - return ctxerr.Wrap(ctx, err, "batch get certificate template statuses") + return 0, ctxerr.Wrap(ctx, err, "batch get certificate template statuses") } } @@ -133,7 +174,7 @@ func (r *profileReconciler) ReconcileProfiles(ctx context.Context) error { toRemove := toRemoveByHostUUID[hostUUID] bulkProfs, err := r.sendHostProfiles(ctx, hostUUID, toInstallProfs, toRemove, profilesContents, certAliases, allCertStatuses[hostUUID]) if err != nil { - return ctxerr.Wrapf(ctx, err, "send profiles for host %s", hostUUID) + return 0, ctxerr.Wrapf(ctx, err, "send profiles for host %s", hostUUID) } bulkHostProfs = append(bulkHostProfs, bulkProfs...) delete(toRemoveByHostUUID, hostUUID) @@ -143,15 +184,15 @@ func (r *profileReconciler) ReconcileProfiles(ctx context.Context) error { for hostUUID, toRemove := range toRemoveByHostUUID { bulkProfs, err := r.sendHostProfiles(ctx, hostUUID, nil, toRemove, nil, nil, nil) if err != nil { - return ctxerr.Wrapf(ctx, err, "send profiles for host %s", hostUUID) + return 0, ctxerr.Wrapf(ctx, err, "send profiles for host %s", hostUUID) } bulkHostProfs = append(bulkHostProfs, bulkProfs...) } if err := r.DS.BulkUpsertMDMAndroidHostProfiles(ctx, bulkHostProfs); err != nil { - return ctxerr.Wrap(ctx, err, "bulk upsert android host profiles") + return 0, ctxerr.Wrap(ctx, err, "bulk upsert android host profiles") } - return nil + return hostCount, nil } func (r *profileReconciler) sendHostProfiles( diff --git a/server/mdm/android/service/profiles_test.go b/server/mdm/android/service/profiles_test.go index 1b8deaa290..f582b69bd7 100644 --- a/server/mdm/android/service/profiles_test.go +++ b/server/mdm/android/service/profiles_test.go @@ -122,13 +122,13 @@ func testNoHost(t *testing.T, ds fleet.Datastore, client *mock.Client, reconcile } // no host, so no calls to the Android API - err := reconciler.ReconcileProfiles(ctx) + _, err := reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) // run again, still nothing - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) @@ -149,13 +149,13 @@ func testHostsWithoutProfile(t *testing.T, ds fleet.Datastore, client *mock.Clie createAndroidHost(t, ds, 2) // nothing to process, no profiles missing nor extraneous - err := reconciler.ReconcileProfiles(ctx) + _, err := reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) // run again, still nothing to process - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) @@ -183,7 +183,7 @@ func testHostsWithProfile(t *testing.T, ds fleet.Datastore, client *mock.Client, require.NoError(t, err) // profile gets delivered to both hosts - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.True(t, client.EnterprisesPoliciesPatchFuncInvoked) client.EnterprisesPoliciesPatchFuncInvoked = false @@ -196,7 +196,7 @@ func testHostsWithProfile(t *testing.T, ds fleet.Datastore, client *mock.Client, }) // run again, nothing to process as everything is pending - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) @@ -228,7 +228,7 @@ func testHostsWithConflictProfile(t *testing.T, ds fleet.Datastore, client *mock require.NoError(t, err) // profiles get delivered to both hosts, but p1 is failed - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.True(t, client.EnterprisesPoliciesPatchFuncInvoked) client.EnterprisesPoliciesPatchFuncInvoked = false @@ -243,7 +243,7 @@ func testHostsWithConflictProfile(t *testing.T, ds fleet.Datastore, client *mock }) // run again, nothing to process as everything is pending/failed - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) @@ -283,7 +283,7 @@ func testHostsWithMultiOverrideProfile(t *testing.T, ds fleet.Datastore, client require.NoError(t, err) // profiles get delivered to h1 only - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.True(t, client.EnterprisesPoliciesPatchFuncInvoked) client.EnterprisesPoliciesPatchFuncInvoked = false @@ -297,7 +297,7 @@ func testHostsWithMultiOverrideProfile(t *testing.T, ds fleet.Datastore, client }) // run again, nothing to process as everything is pending/failed - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) @@ -324,7 +324,7 @@ func testHostsWithAPIFailures(t *testing.T, ds fleet.Datastore, client *mock.Cli require.NoError(t, err) for i := range 3 { - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.True(t, client.EnterprisesPoliciesPatchFuncInvoked) client.EnterprisesPoliciesPatchFuncInvoked = false @@ -338,7 +338,7 @@ func testHostsWithAPIFailures(t *testing.T, ds fleet.Datastore, client *mock.Cli } // next run marks as failed - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) @@ -349,7 +349,7 @@ func testHostsWithAPIFailures(t *testing.T, ds fleet.Datastore, client *mock.Cli }) // next run has nothing to do - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) @@ -365,7 +365,7 @@ func testHostsWithAPIFailures(t *testing.T, ds fleet.Datastore, client *mock.Cli return policy, nil } - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.True(t, client.EnterprisesPoliciesPatchFuncInvoked) client.EnterprisesPoliciesPatchFuncInvoked = false @@ -404,7 +404,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie p1Checksum := getAndroidProfileChecksum(t, ds, p1.ProfileUUID) // profiles get delivered - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.True(t, client.EnterprisesPoliciesPatchFuncInvoked) client.EnterprisesPoliciesPatchFuncInvoked = false @@ -417,7 +417,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie }) // run again, nothing to process - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) @@ -431,7 +431,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie require.NoError(t, err) // run again, nothing to process - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) @@ -452,7 +452,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie require.NoError(t, err) // profile gets re-delivered - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.True(t, client.EnterprisesPoliciesPatchFuncInvoked) client.EnterprisesPoliciesPatchFuncInvoked = false @@ -471,7 +471,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie p2Checksum := getAndroidProfileChecksum(t, ds, p2.ProfileUUID) // profiles get re-delivered - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.True(t, client.EnterprisesPoliciesPatchFuncInvoked) client.EnterprisesPoliciesPatchFuncInvoked = false @@ -486,7 +486,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie }) // run again, nothing to process - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) @@ -500,7 +500,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie require.NoError(t, err) // run again, nothing to process - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) @@ -516,7 +516,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie } // profiles get re-delivered - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.True(t, client.EnterprisesPoliciesPatchFuncInvoked) client.EnterprisesPoliciesPatchFuncInvoked = false @@ -532,7 +532,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie }) // run again, nothing to process - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) @@ -552,7 +552,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie }) // run again, nothing to process - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) @@ -609,7 +609,7 @@ func testHostsWithLabelProfiles(t *testing.T, ds fleet.Datastore, client *mock.C } // currently only the no-label profile is applied - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.True(t, client.EnterprisesPoliciesPatchFuncInvoked) client.EnterprisesPoliciesPatchFuncInvoked = false @@ -631,7 +631,7 @@ func testHostsWithLabelProfiles(t *testing.T, ds fleet.Datastore, client *mock.C // no-label and exclude any are applied version++ - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.True(t, client.EnterprisesPoliciesPatchFuncInvoked) client.EnterprisesPoliciesPatchFuncInvoked = false @@ -653,7 +653,7 @@ func testHostsWithLabelProfiles(t *testing.T, ds fleet.Datastore, client *mock.C // no-label, exclude any and the respective include profiles are applied version++ - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.True(t, client.EnterprisesPoliciesPatchFuncInvoked) client.EnterprisesPoliciesPatchFuncInvoked = false @@ -676,7 +676,7 @@ func testHostsWithLabelProfiles(t *testing.T, ds fleet.Datastore, client *mock.C // this only affects h1, h2 version is unchanged h2Version := version version++ - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.True(t, client.EnterprisesPoliciesPatchFuncInvoked) client.EnterprisesPoliciesPatchFuncInvoked = false @@ -694,7 +694,7 @@ func testHostsWithLabelProfiles(t *testing.T, ds fleet.Datastore, client *mock.C }) // run again, nothing to process - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) require.False(t, client.EnterprisesPoliciesPatchFuncInvoked) require.False(t, client.EnterprisesDevicesPatchFuncInvoked) @@ -1314,7 +1314,7 @@ func testONCWithheldUntilCertVerified(t *testing.T, ds fleet.Datastore, client * require.NoError(t, err) // --- Phase 1: cert is pending, ONC should be withheld, non-ONC applied --- - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) assertHostProfiles(t, ds, []*fleet.MDMAndroidProfilePayload{ @@ -1346,7 +1346,7 @@ func testONCWithheldUntilCertVerified(t *testing.T, ds fleet.Datastore, client * client.EnterprisesPoliciesPatchFuncInvoked = false client.EnterprisesDevicesPatchFuncInvoked = false - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) // Both profiles should now be applied (included in policy, with request UUIDs) @@ -1383,7 +1383,7 @@ func testONCWithheldUntilCertVerified(t *testing.T, ds fleet.Datastore, client * }) // cert is "delivered" (not terminal), ONC should be withheld - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) phase3Profiles, err := ds.GetHostMDMAndroidProfiles(ctx, host.UUID) @@ -1403,7 +1403,7 @@ func testONCWithheldUntilCertVerified(t *testing.T, ds fleet.Datastore, client * ) return err }) - err = reconciler.ReconcileProfiles(ctx) + _, err = reconciler.ReconcileProfiles(ctx, "", 0) require.NoError(t, err) // Both profiles applied (cert is terminally failed, ONC released) diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index d1d7cc02a0..71694e6fdc 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -1888,7 +1888,11 @@ type GetHostMDMAndroidProfilesFunc func(ctx context.Context, hostUUID string) ([ type NewAndroidPolicyRequestFunc func(ctx context.Context, req *android.MDMAndroidPolicyRequest) error -type ListMDMAndroidProfilesToSendFunc func(ctx context.Context) ([]*fleet.MDMAndroidProfilePayload, []*fleet.MDMAndroidProfilePayload, error) +type ListMDMAndroidProfilesToSendFunc func(ctx context.Context, cursor string, batchSize int) ([]*fleet.MDMAndroidProfilePayload, []*fleet.MDMAndroidProfilePayload, error) + +type GetMDMAndroidReconcileCursorFunc func(ctx context.Context) (string, error) + +type SetMDMAndroidReconcileCursorFunc func(ctx context.Context, cursor string) error type GetMDMAndroidProfilesContentsFunc func(ctx context.Context, uuids []string) (map[string]json.RawMessage, error) @@ -4900,6 +4904,12 @@ type DataStore struct { ListMDMAndroidProfilesToSendFunc ListMDMAndroidProfilesToSendFunc ListMDMAndroidProfilesToSendFuncInvoked bool + GetMDMAndroidReconcileCursorFunc GetMDMAndroidReconcileCursorFunc + GetMDMAndroidReconcileCursorFuncInvoked bool + + SetMDMAndroidReconcileCursorFunc SetMDMAndroidReconcileCursorFunc + SetMDMAndroidReconcileCursorFuncInvoked bool + GetMDMAndroidProfilesContentsFunc GetMDMAndroidProfilesContentsFunc GetMDMAndroidProfilesContentsFuncInvoked bool @@ -11742,11 +11752,25 @@ func (s *DataStore) NewAndroidPolicyRequest(ctx context.Context, req *android.MD return s.NewAndroidPolicyRequestFunc(ctx, req) } -func (s *DataStore) ListMDMAndroidProfilesToSend(ctx context.Context) ([]*fleet.MDMAndroidProfilePayload, []*fleet.MDMAndroidProfilePayload, error) { +func (s *DataStore) ListMDMAndroidProfilesToSend(ctx context.Context, cursor string, batchSize int) ([]*fleet.MDMAndroidProfilePayload, []*fleet.MDMAndroidProfilePayload, error) { s.mu.Lock() s.ListMDMAndroidProfilesToSendFuncInvoked = true s.mu.Unlock() - return s.ListMDMAndroidProfilesToSendFunc(ctx) + return s.ListMDMAndroidProfilesToSendFunc(ctx, cursor, batchSize) +} + +func (s *DataStore) GetMDMAndroidReconcileCursor(ctx context.Context) (string, error) { + s.mu.Lock() + s.GetMDMAndroidReconcileCursorFuncInvoked = true + s.mu.Unlock() + return s.GetMDMAndroidReconcileCursorFunc(ctx) +} + +func (s *DataStore) SetMDMAndroidReconcileCursor(ctx context.Context, cursor string) error { + s.mu.Lock() + s.SetMDMAndroidReconcileCursorFuncInvoked = true + s.mu.Unlock() + return s.SetMDMAndroidReconcileCursorFunc(ctx, cursor) } func (s *DataStore) GetMDMAndroidProfilesContents(ctx context.Context, uuids []string) (map[string]json.RawMessage, error) { diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index 36684f83a6..968ae67d82 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -485,7 +485,7 @@ func (s *integrationMDMTestSuite) SetupSuite() { err := android_service.ReconcileProfilesWithClient(ctx, ds, logger, "", androidMockClient, config.AndroidAgentConfig{ Package: "com.fleetdm.agent", SigningSHA256: "abc123def456", - }) + }, 0) require.NoError(s.T(), err) return err }),