Throttle requests to AMAPI during profile reconcilation (#47223)

**Related issue:** Resolves #41910

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.
- [x] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements), JS
inline code is prevented especially for url redirects, and untrusted
data interpolated into shell scripts/commands is validated against shell
metacharacters.

## Testing

- [x] Added/updated automated tests
- [x] QA'd all new/changed functionality manually

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added a configurable env var to limit Android MDM profile
reconciliation batch size (FLEET_MDM_ANDROID_PROFILES_BATCH_SIZE;
default 1000).
* Reconciliation now processes hosts in cursor-based, batched windows
and persists a reconciliation cursor to resume/advance work, reducing
peak API load and enabling pagination.
* **Tests**
* Added validation tests for the batch-size config and tests verifying
cursor-based pagination and processing.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Konstantin Sykulev
2026-06-12 13:13:29 -05:00
committed by GitHub
parent a3f07e3752
commit 2ad76714ce
13 changed files with 376 additions and 91 deletions
+1
View File
@@ -0,0 +1 @@
- Added configurable batch size `FLEET_MDM_ANDROID_BATCH_SIZE` (default: 1000 hosts) for Android MDM operations to prevent overwhelming the Google Android Management API.
+2 -1
View File
@@ -1930,6 +1930,7 @@ func newAndroidMDMProfileManagerSchedule(
logger *slog.Logger,
licenseKey string,
androidAgentConfig config.AndroidAgentConfig,
batchSize int,
) (*schedule.Schedule, error) {
const (
name = string(fleet.CronMDMAndroidProfileManager)
@@ -1941,7 +1942,7 @@ func newAndroidMDMProfileManagerSchedule(
ctx, name, instanceID, defaultInterval, ds, ds,
schedule.WithLogger(logger),
schedule.WithJob("manage_android_profiles", func(ctx context.Context) error {
return android_svc.ReconcileProfiles(ctx, ds, logger, licenseKey, androidAgentConfig)
return android_svc.ReconcileProfiles(ctx, ds, logger, licenseKey, androidAgentConfig, batchSize)
}),
)
+2
View File
@@ -443,6 +443,7 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev
// Declare svc early so the closure below can capture it.
var svc fleet.Service
config.MDM.AndroidAgent.Validate(initFatal)
config.MDM.ValidateAndroidBatchSize(initFatal)
androidSvc, err := android_service.NewService(
ctx,
logger,
@@ -829,6 +830,7 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev
logger,
config.License.Key, // NOTE: this requires the license key, not the parsed *LicenseInfo available in the ctx
config.MDM.AndroidAgent,
config.MDM.AndroidBatchSize,
)
}); err != nil {
initFatal(err, "failed to register mdm_android_profile_manager schedule")
+13 -1
View File
@@ -926,13 +926,22 @@ type MDMConfig struct {
EnableCustomFileVault bool `yaml:"enable_custom_filevault"`
AllowAllDeclarations bool `yaml:"allow_all_declarations"`
AndroidAgent AndroidAgentConfig `yaml:"android_agent"`
AndroidAgent AndroidAgentConfig `yaml:"android_agent"`
AndroidBatchSize int `yaml:"android_batch_size"`
}
func (m MDMConfig) IsCustomFileVaultEnabled() bool {
return m.EnableCustomOSUpdatesAndFileVault || m.EnableCustomFileVault
}
// ValidateAndroidBatchSize checks that the configured batch size is non-negative.
func (m MDMConfig) ValidateAndroidBatchSize(initFatal func(err error, msg string)) {
if m.AndroidBatchSize < 0 {
initFatal(errors.New("mdm.android_batch_size must be non-negative (0 = no limit)"),
"Android MDM configuration")
}
}
// AndroidAgentConfig holds configuration for the Fleet Android agent.
type AndroidAgentConfig struct {
// Package is the package name for the Fleet Android agent.
@@ -1758,6 +1767,8 @@ func (man Manager) addConfigs() {
man.addConfigString("mdm.android_agent.signing_sha256", "x+IyvrwVbQEBYV/ojWmLavJE0VIZE1RAT2JmxeI5sFw=", "Signing certificate SHA256 fingerprint for the Fleet Android agent")
man.hideConfig("mdm.android_agent.package")
man.hideConfig("mdm.android_agent.signing_sha256")
man.addConfigInt("mdm.android_batch_size", 1000, "Maximum number of hosts per batch for Android MDM API operations (1000 default; 0 = no limit)")
man.hideConfig("mdm.android_batch_size")
// Calendar integration
man.addConfigDuration(
@@ -2091,6 +2102,7 @@ func (man Manager) LoadConfig() FleetConfig {
Package: man.getConfigString("mdm.android_agent.package"),
SigningSHA256: man.getConfigString("mdm.android_agent.signing_sha256"),
},
AndroidBatchSize: man.getConfigInt("mdm.android_batch_size"),
},
Calendar: CalendarConfig{
Periodicity: man.getConfigDuration("calendar.periodicity"),
+21
View File
@@ -803,6 +803,27 @@ func TestAndroidAgentConfigValidate(t *testing.T) {
})
}
func TestAndroidBatchSizeValidate(t *testing.T) {
t.Parallel()
t.Run("valid when positive", func(t *testing.T) {
cfg := MDMConfig{AndroidBatchSize: 1000}
cfg.ValidateAndroidBatchSize(func(err error, msg string) { t.Fatalf("unexpected error: %v", err) })
})
t.Run("valid when zero", func(t *testing.T) {
cfg := MDMConfig{AndroidBatchSize: 0}
cfg.ValidateAndroidBatchSize(func(err error, msg string) { t.Fatalf("unexpected error: %v", err) })
})
t.Run("invalid when negative", func(t *testing.T) {
cfg := MDMConfig{AndroidBatchSize: -1}
called := false
cfg.ValidateAndroidBatchSize(func(err error, msg string) { called = true })
require.True(t, called)
})
}
func TestServerConfigWithH2C(t *testing.T) {
ctx := context.Background()
+41 -5
View File
@@ -1313,6 +1313,18 @@ const androidApplicableProfilesQuery = `
count_host_updated_after_labels = 0
`
// GetMDMAndroidReconcileCursor is a no-op on the bare mysql.Datastore;
// the mysqlredis wrapper backs it with Redis.
func (ds *Datastore) GetMDMAndroidReconcileCursor(_ context.Context) (string, error) {
return "", nil
}
// SetMDMAndroidReconcileCursor is a no-op on the bare mysql.Datastore;
// the mysqlredis wrapper writes to Redis.
func (ds *Datastore) SetMDMAndroidReconcileCursor(_ context.Context, _ string) error {
return nil
}
// ListMDMAndroidProfilesToSend is the android platform equivalent to
// ListMDMAppleProfilesToInstall/Remove and
// ListMDMWindowsProfilesToInstall/Remove. It plays a similar role but is quite
@@ -1336,12 +1348,28 @@ const androidApplicableProfilesQuery = `
//
// See https://github.com/fleetdm/fleet/issues/32032#issuecomment-3229548389
// for more details on the rationale of that approach.
func (ds *Datastore) ListMDMAndroidProfilesToSend(ctx context.Context) ([]*fleet.MDMAndroidProfilePayload, []*fleet.MDMAndroidProfilePayload, error) {
func (ds *Datastore) ListMDMAndroidProfilesToSend(ctx context.Context, cursor string, batchSize int) ([]*fleet.MDMAndroidProfilePayload, []*fleet.MDMAndroidProfilePayload, error) {
var toApplyProfiles, toRemoveProfiles []*fleet.MDMAndroidProfilePayload
err := ds.withTx(ctx, func(tx sqlx.ExtContext) error {
var installCursorPred, removeCursorPred string
var args []any
if cursor != "" {
installCursorPred = "AND ds.host_uuid > ?"
removeCursorPred = "AND hmap.host_uuid > ?"
args = []any{cursor, fleet.MDMOperationTypeRemove, fleet.MDMDeliveryPending, cursor}
} else {
args = []any{fleet.MDMOperationTypeRemove, fleet.MDMDeliveryPending}
}
var limitClause string
if batchSize > 0 {
limitClause = fmt.Sprintf("LIMIT %d", batchSize)
}
hostsWithChangesStmt := fmt.Sprintf(`
WITH ds AS ( %s )
SELECT host_uuid FROM (
SELECT
DISTINCT ds.host_uuid
FROM ds
@@ -1362,6 +1390,7 @@ func (ds *Datastore) ListMDMAndroidProfilesToSend(ctx context.Context) ([]*fleet
-- profile needs retry (status reset to NULL after transient failure)
hmap.status IS NULL
)
%s
UNION
@@ -1382,7 +1411,15 @@ func (ds *Datastore) ListMDMAndroidProfilesToSend(ctx context.Context) ([]*fleet
ds.host_uuid IS NULL AND
-- and it is not in pending remove status (in which case it was processed)
( hmap.operation_type != ? OR COALESCE(hmap.status, '') <> ? )
`, fmt.Sprintf(androidApplicableProfilesQuery, "TRUE", "TRUE", "TRUE", "TRUE", "TRUE", "TRUE"))
%s
) AS all_changes
ORDER BY host_uuid
%s
`, fmt.Sprintf(androidApplicableProfilesQuery, "TRUE", "TRUE", "TRUE", "TRUE", "TRUE", "TRUE"),
installCursorPred,
removeCursorPred,
limitClause,
)
// NOTE: we explicitly don't "ignore" profiles to remove based on broken labels,
// because of how Android profiles are applied vs other platforms (ignoring
@@ -1397,8 +1434,7 @@ func (ds *Datastore) ListMDMAndroidProfilesToSend(ctx context.Context) ([]*fleet
// see https://github.com/fleetdm/fleet/issues/25557#issuecomment-3246496873
var hostUUIDs []string
if err := sqlx.SelectContext(ctx, tx, &hostUUIDs, hostsWithChangesStmt,
fleet.MDMOperationTypeRemove, fleet.MDMDeliveryPending); err != nil {
if err := sqlx.SelectContext(ctx, tx, &hostUUIDs, hostsWithChangesStmt, args...); err != nil {
return ctxerr.Wrap(ctx, err, "list android hosts with profile changes")
}
@@ -1886,7 +1922,7 @@ func (ds *Datastore) bulkSetPendingMDMAndroidHostProfilesDB(
return false, nil
}
profilesToInstall, profilesToRemove, err := ds.ListMDMAndroidProfilesToSend(ctx)
profilesToInstall, profilesToRemove, err := ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
if err != nil {
return false, ctxerr.Wrap(ctx, err, "list android profiles to send")
}
+119 -32
View File
@@ -1,9 +1,12 @@
package mysql
import (
"cmp"
"context"
"encoding/json"
"fmt"
"maps"
"slices"
"strings"
"testing"
"time"
@@ -39,6 +42,7 @@ func TestAndroid(t *testing.T) {
{"ListMDMAndroidProfilesToSend", testListMDMAndroidProfilesToSend},
{"ListMDMAndroidProfilesToSend_WithExcludeAny", testListMDMAndroidProfilesToSendWithExcludeAny},
{"ListMDMAndroidProfilesToSend_WithCombinedLabels", testListMDMAndroidProfilesToSendWithCombinedLabels},
{"ListMDMAndroidProfilesToSend_Cursor", testListMDMAndroidProfilesToSendCursor},
{"GetMDMAndroidProfilesContents", testGetMDMAndroidProfilesContents},
{"BulkUpsertMDMAndroidHostProfiles", testBulkUpsertMDMAndroidHostProfiles},
{"BulkUpsertMDMAndroidHostProfiles", testBulkUpsertMDMAndroidHostProfiles2},
@@ -1347,7 +1351,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
}
// without any profile, should return empty
profs, toRemoveProfs, err := ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err := ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, profs)
require.Empty(t, toRemoveProfs)
@@ -1369,7 +1373,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
profChecksum := getAndroidProfileChecksum(t, ds, p1.ProfileUUID)
// both no-team profiles should be applicable to both hosts
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 4)
@@ -1385,7 +1389,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
require.NoError(t, err)
// profiles for host 1 change to p3
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 3)
@@ -1404,7 +1408,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
require.NoError(t, err)
// no change, host is not a member of both labels
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 3)
@@ -1419,7 +1423,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
require.NoError(t, err)
// no change, host is not a member of both labels
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 3)
@@ -1434,7 +1438,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
require.NoError(t, err)
// now p4 is applicable to host 0
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 4)
@@ -1454,7 +1458,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
require.NoError(t, err)
// no change, host 0 not a member yet
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 4)
@@ -1470,7 +1474,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
require.NoError(t, err)
// now p5 is applicable to host 0
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 5)
@@ -1491,7 +1495,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
require.NoError(t, err)
// no change, label membership was not updated after labels created
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 5)
@@ -1510,7 +1514,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
require.NoError(t, err)
// host 0 is _not_ a member of the excluded labels, so p6 is applicable
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 6)
@@ -1528,7 +1532,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
require.NoError(t, err)
// p6 is not applicable anymore
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 5)
@@ -1549,7 +1553,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
require.NoError(t, err)
// it is not included in noProfHosts as it has p3
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 6)
@@ -1571,7 +1575,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
})
// host 2 is not included in the results as it has p3 installed
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 5)
@@ -1588,7 +1592,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
require.NoError(t, err)
// host 2 is now a host with no profile (profile 3 needs to be cleared), host 1 is unlisted as it didn't have p3 installed
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.ElementsMatch(t, []*fleet.MDMAndroidProfilePayload{
{ProfileUUID: p3.ProfileUUID, HostUUID: hosts[2].UUID, ProfileName: p3.Name, Checksum: profChecksum},
@@ -1607,7 +1611,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
return err
})
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 4)
@@ -1623,7 +1627,7 @@ func testListMDMAndroidProfilesToSend(t *testing.T, ds *Datastore) {
_, err := q.ExecContext(ctx, `UPDATE host_mdm SET enrolled=0 WHERE host_id=?`, hosts[0].ID)
return err
})
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, profs)
require.Empty(t, toRemoveProfs)
@@ -1649,7 +1653,7 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore)
}
// without any profile, should return empty
profs, toRemoveProfs, err := ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err := ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, profs)
require.Empty(t, toRemoveProfs)
@@ -1682,7 +1686,7 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore)
profChecksum := getAndroidProfileChecksum(t, ds, p1.ProfileUUID)
// p2 becomes immediately applicable because it only has a manual label
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 1)
@@ -1697,7 +1701,7 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore)
require.NoError(t, err)
// host 0 dynamic labels now apply, and this host is _not_ a member of the excluded labels, so p1, p2 and p3 are now applicable
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 3)
@@ -1727,7 +1731,7 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore)
require.NoError(t, err)
// p5 becomes immediately applicable to host 1 because it only has a manual label
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 4)
@@ -1746,7 +1750,7 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore)
require.NoError(t, ds.AddHostsToTeam(ctx, fleet.NewAddHostsToTeamParams(&tm.ID, []uint{hosts[1].ID})))
hosts[1].TeamID = &tm.ID
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 6)
@@ -1763,7 +1767,7 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore)
_, _, err = ds.UpdateLabelMembershipByHostIDs(ctx, *lblExclAny2, []uint{hosts[0].ID}, fleet.TeamFilter{})
require.NoError(t, err)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 4)
@@ -1780,7 +1784,7 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore)
_, _, err = ds.UpdateLabelMembershipByHostIDs(ctx, *lblExclAny1, []uint{hosts[0].ID, hosts[1].ID}, fleet.TeamFilter{})
require.NoError(t, err)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.Len(t, profs, 1)
@@ -1789,6 +1793,89 @@ func testListMDMAndroidProfilesToSendWithExcludeAny(t *testing.T, ds *Datastore)
}, profs)
}
func testListMDMAndroidProfilesToSendCursor(t *testing.T, ds *Datastore) {
test.AddBuiltinLabels(t, ds)
ctx := t.Context()
// Create 5 hosts with predictable UUIDs for cursor ordering.
hosts := make([]*fleet.Host, 5)
for i := range hosts {
androidHost := createAndroidHost(fmt.Sprintf("cursor-host-%02d", i))
newHost, err := ds.NewAndroidHost(ctx, androidHost, false)
require.NoError(t, err)
hosts[i] = newHost.Host
}
// Sort by UUID so we can predict cursor order.
slices.SortFunc(hosts, func(a, b *fleet.Host) int {
return cmp.Compare(a.UUID, b.UUID)
})
// Add a profile so all 5 hosts have pending work.
_, err := ds.NewMDMAndroidConfigProfile(ctx, *androidProfileForTest("cursor-test-profile"))
require.NoError(t, err)
// No cursor, no limit — returns all 5 hosts.
allProfs, _, err := ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
allHostUUIDs := make(map[string]struct{})
for _, p := range allProfs {
allHostUUIDs[p.HostUUID] = struct{}{}
}
require.Len(t, allHostUUIDs, 5)
// Batch 1: limit 2 hosts, no cursor.
batch1Profs, _, err := ds.ListMDMAndroidProfilesToSend(ctx, "", 2)
require.NoError(t, err)
batch1Hosts := make(map[string]struct{})
for _, p := range batch1Profs {
batch1Hosts[p.HostUUID] = struct{}{}
}
require.Len(t, batch1Hosts, 2, "batch 1 should return exactly 2 hosts")
// Hosts should be the first 2 in sorted order.
sorted1 := slices.Sorted(maps.Keys(batch1Hosts))
require.Equal(t, hosts[0].UUID, sorted1[0])
require.Equal(t, hosts[1].UUID, sorted1[1])
// Batch 2: cursor past the last host of batch 1, limit 2.
cursor := sorted1[len(sorted1)-1]
batch2Profs, _, err := ds.ListMDMAndroidProfilesToSend(ctx, cursor, 2)
require.NoError(t, err)
batch2Hosts := make(map[string]struct{})
for _, p := range batch2Profs {
batch2Hosts[p.HostUUID] = struct{}{}
}
require.Len(t, batch2Hosts, 2, "batch 2 should return exactly 2 hosts")
// No overlap with batch 1, and all hosts should be > cursor.
sorted2 := slices.Sorted(maps.Keys(batch2Hosts))
for _, uuid := range sorted2 {
require.Greater(t, uuid, cursor, "batch 2 hosts must be after cursor")
_, overlap := batch1Hosts[uuid]
require.False(t, overlap, "batch 2 must not overlap with batch 1")
}
// Batch 3: cursor past batch 2, limit 2 — should return the remaining 1 host.
cursor = sorted2[len(sorted2)-1]
batch3Profs, _, err := ds.ListMDMAndroidProfilesToSend(ctx, cursor, 2)
require.NoError(t, err)
batch3Hosts := make(map[string]struct{})
for _, p := range batch3Profs {
batch3Hosts[p.HostUUID] = struct{}{}
}
require.Len(t, batch3Hosts, 1, "batch 3 should return the remaining 1 host")
sorted3 := slices.Sorted(maps.Keys(batch3Hosts))
require.Greater(t, sorted3[0], cursor, "batch 3 host must be after cursor")
// Batch 4: cursor past batch 3 — should return empty (end of pass).
cursor = sorted3[0]
batch4Profs, _, err := ds.ListMDMAndroidProfilesToSend(ctx, cursor, 2)
require.NoError(t, err)
require.Empty(t, batch4Profs, "no more hosts after end of universe")
}
func testListMDMAndroidProfilesToSendWithCombinedLabels(t *testing.T, ds *Datastore) {
test.AddBuiltinLabels(t, ds)
ctx := t.Context()
@@ -1823,7 +1910,7 @@ func testListMDMAndroidProfilesToSendWithCombinedLabels(t *testing.T, ds *Datast
profChecksum := getAndroidProfileChecksum(t, ds, pCombinedAll.ProfileUUID)
// host is not a member of any label → neither profile applies
profs, toRemove, err := ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemove, err := ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemove)
require.Empty(t, profs)
@@ -1832,7 +1919,7 @@ func testListMDMAndroidProfilesToSendWithCombinedLabels(t *testing.T, ds *Datast
err = ds.AddLabelsToHost(ctx, h.ID, []uint{inclAllLbl.ID, inclAllLbl2.ID, inclAnyLbl.ID})
require.NoError(t, err)
profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemove)
require.Len(t, profs, 2)
@@ -1845,7 +1932,7 @@ func testListMDMAndroidProfilesToSendWithCombinedLabels(t *testing.T, ds *Datast
err = ds.AddLabelsToHost(ctx, h.ID, []uint{exclLbl.ID})
require.NoError(t, err)
profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemove)
require.Empty(t, profs)
@@ -1854,7 +1941,7 @@ func testListMDMAndroidProfilesToSendWithCombinedLabels(t *testing.T, ds *Datast
err = ds.RemoveLabelsFromHost(ctx, h.ID, []uint{exclLbl.ID})
require.NoError(t, err)
profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemove)
require.Len(t, profs, 2)
@@ -1867,7 +1954,7 @@ func testListMDMAndroidProfilesToSendWithCombinedLabels(t *testing.T, ds *Datast
err = ds.RemoveLabelsFromHost(ctx, h.ID, []uint{inclAllLbl2.ID})
require.NoError(t, err)
profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx)
profs, toRemove, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemove)
require.Len(t, profs, 1)
@@ -1976,7 +2063,7 @@ func testBulkUpsertMDMAndroidHostProfilesN(t *testing.T, ds *Datastore, batchSiz
ds.testUpsertMDMDesiredProfilesBatchSize = batchSize
t.Cleanup(func() { ds.testUpsertMDMDesiredProfilesBatchSize = 0 })
hostProfiles, toRemoveProfs, err := ds.ListMDMAndroidProfilesToSend(ctx)
hostProfiles, toRemoveProfs, err := ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.ElementsMatch(t, []*fleet.MDMAndroidProfilePayload{
@@ -2019,7 +2106,7 @@ func testBulkUpsertMDMAndroidHostProfilesN(t *testing.T, ds *Datastore, batchSiz
})
require.NoError(t, err)
hostProfiles, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
hostProfiles, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.ElementsMatch(t, []*fleet.MDMAndroidProfilePayload{
@@ -2061,7 +2148,7 @@ func testBulkUpsertMDMAndroidHostProfilesN(t *testing.T, ds *Datastore, batchSiz
})
require.NoError(t, err)
hostProfiles, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
hostProfiles, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.Empty(t, toRemoveProfs)
require.ElementsMatch(t, []*fleet.MDMAndroidProfilePayload{
@@ -2076,7 +2163,7 @@ func testBulkUpsertMDMAndroidHostProfilesN(t *testing.T, ds *Datastore, batchSiz
err = ds.DeleteMDMAndroidConfigProfile(ctx, profiles[2].ProfileUUID)
require.NoError(t, err)
hostProfiles, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx)
hostProfiles, toRemoveProfs, err = ds.ListMDMAndroidProfilesToSend(ctx, "", 0)
require.NoError(t, err)
require.ElementsMatch(t, []*fleet.MDMAndroidProfilePayload{
{ProfileUUID: profiles[2].ProfileUUID, HostUUID: hosts[2].UUID, ProfileName: profiles[2].Name, Checksum: profChecksum},
@@ -0,0 +1,49 @@
package mysqlredis
import (
"context"
"errors"
"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
"github.com/fleetdm/fleet/v4/server/datastore/redis"
redigo "github.com/gomodule/redigo/redis"
)
// androidReconCursorKey is the Redis key holding the host_uuid cursor for
// the batched Android MDM profile reconciler.
const androidReconCursorKey = "mdm:android:recon_cursor"
// GetMDMAndroidReconcileCursor returns the persisted host_uuid cursor used by
// the batched Android MDM reconciliation cron to bound per-tick work.
//
// Returns "" if the key is unset (fresh deployment, Redis flushed, or full
// pass complete). Loss of this key is harmless: the cron resumes from the
// beginning. The desired-state diff is recomputed every tick, so re-
// processing converges naturally.
func (d *Datastore) GetMDMAndroidReconcileCursor(ctx context.Context) (string, error) {
conn := redis.ConfigureDoer(d.pool, d.pool.Get())
defer conn.Close()
cursor, err := redigo.String(conn.Do("GET", androidReconCursorKey))
switch {
case err == nil:
return cursor, nil
case errors.Is(err, redigo.ErrNil):
return "", nil
default:
return "", ctxerr.Wrap(ctx, err, "get android MDM reconcile cursor")
}
}
// SetMDMAndroidReconcileCursor persists the host_uuid cursor used by the
// batched Android MDM reconciliation cron. An empty string indicates a full
// pass has completed; the next tick will start from the beginning.
func (d *Datastore) SetMDMAndroidReconcileCursor(ctx context.Context, cursor string) error {
conn := redis.ConfigureDoer(d.pool, d.pool.Get())
defer conn.Close()
if _, err := conn.Do("SET", androidReconCursorKey, cursor); err != nil {
return ctxerr.Wrap(ctx, err, "set android MDM reconcile cursor")
}
return nil
}
+12 -1
View File
@@ -3131,7 +3131,18 @@ type Datastore interface {
// ListMDMAndroidProfilesToSend lists the Android hosts that need to have
// their configuration profiles (Android policy) sent. It returns two lists,
// the list of profiles to apply and the list of profiles to remove.
ListMDMAndroidProfilesToSend(ctx context.Context) ([]*MDMAndroidProfilePayload, []*MDMAndroidProfilePayload, error)
// When cursor is non-empty, only hosts with host_uuid > cursor are
// considered. When batchSize > 0, at most batchSize distinct hosts are
// returned.
ListMDMAndroidProfilesToSend(ctx context.Context, cursor string, batchSize int) ([]*MDMAndroidProfilePayload, []*MDMAndroidProfilePayload, error)
// GetMDMAndroidReconcileCursor returns the persisted host_uuid cursor
// used by the Android MDM reconciliation cron to bound per-tick work.
GetMDMAndroidReconcileCursor(ctx context.Context) (string, error)
// SetMDMAndroidReconcileCursor persists the host_uuid cursor used by
// the Android MDM reconciliation cron. See GetMDMAndroidReconcileCursor.
SetMDMAndroidReconcileCursor(ctx context.Context, cursor string) error
// GetMDMAndroidProfilesContents retrieves the contents of the Android
// profiles with the specified UUIDs.
+55 -14
View File
@@ -19,13 +19,13 @@ import (
"google.golang.org/api/androidmanagement/v1"
)
func ReconcileProfiles(ctx context.Context, ds fleet.Datastore, logger *slog.Logger, licenseKey string, androidAgentConfig config.AndroidAgentConfig) error {
return ReconcileProfilesWithClient(ctx, ds, logger, licenseKey, nil, androidAgentConfig)
func ReconcileProfiles(ctx context.Context, ds fleet.Datastore, logger *slog.Logger, licenseKey string, androidAgentConfig config.AndroidAgentConfig, batchSize int) error {
return ReconcileProfilesWithClient(ctx, ds, logger, licenseKey, nil, androidAgentConfig, batchSize)
}
// ReconcileProfilesWithClient is like ReconcileProfiles but allows injecting a custom client for testing.
// If client is nil, a new AMAPI client will be created.
func ReconcileProfilesWithClient(ctx context.Context, ds fleet.Datastore, logger *slog.Logger, licenseKey string, client androidmgmt.Client, androidAgentConfig config.AndroidAgentConfig) error {
func ReconcileProfilesWithClient(ctx context.Context, ds fleet.Datastore, logger *slog.Logger, licenseKey string, client androidmgmt.Client, androidAgentConfig config.AndroidAgentConfig, batchSize int) (err error) {
appConfig, err := ds.AppConfig(ctx)
if err != nil {
return ctxerr.Wrap(ctx, err, "get app config")
@@ -54,6 +54,14 @@ func ReconcileProfilesWithClient(ctx context.Context, ds fleet.Datastore, logger
}
}
// Read the cursor; on error, treat as start-of-pass and continue.
cursor, cursorErr := ds.GetMDMAndroidReconcileCursor(ctx)
if cursorErr != nil {
logger.WarnContext(ctx, "failed to read android MDM reconcile cursor; starting from beginning",
"err", cursorErr)
cursor = ""
}
reconciler := &profileReconciler{
DS: ds,
Enterprise: enterprise,
@@ -61,7 +69,24 @@ func ReconcileProfilesWithClient(ctx context.Context, ds fleet.Datastore, logger
AndroidAgentConfig: androidAgentConfig,
Logger: logger,
}
return reconciler.ReconcileProfiles(ctx)
hostCount, err := reconciler.ReconcileProfiles(ctx, cursor, batchSize)
var nextCursor string
if batchSize > 0 && hostCount >= batchSize {
// reconciler returns hostCount matching the number of distinct
// host UUIDs it processed; advance past them.
nextCursor = reconciler.lastHostUUID
}
// On success, advance the cursor. On failure, leave it where it was
// so the next tick retries the same host window.
if err == nil && cursor != nextCursor {
if cerr := ds.SetMDMAndroidReconcileCursor(ctx, nextCursor); cerr != nil {
logger.WarnContext(ctx, "failed to advance android MDM reconcile cursor", "err", cerr)
}
}
return err
}
// profileReconciler is a struct to facilitate testability, it should not be
@@ -72,6 +97,7 @@ type profileReconciler struct {
Client androidmgmt.Client
AndroidAgentConfig config.AndroidAgentConfig
Logger *slog.Logger
lastHostUUID string
}
func getClientAuthenticationSecret(ctx context.Context, ds fleet.Datastore) (string, error) {
@@ -85,15 +111,15 @@ func getClientAuthenticationSecret(ctx context.Context, ds fleet.Datastore) (str
return string(assets[fleet.MDMAssetAndroidFleetServerSecret].Value), nil
}
func (r *profileReconciler) ReconcileProfiles(ctx context.Context) error {
func (r *profileReconciler) ReconcileProfiles(ctx context.Context, cursor string, batchSize int) (int, error) {
if err := r.reconcileCertificateTemplates(ctx); err != nil {
return ctxerr.Wrap(ctx, err, "reconcile certificate templates")
return 0, ctxerr.Wrap(ctx, err, "reconcile certificate templates")
}
// get the list of hosts that need to have their profiles applied
hostsApplicableProfiles, hostsProfsToRemove, err := r.DS.ListMDMAndroidProfilesToSend(ctx)
hostsApplicableProfiles, hostsProfsToRemove, err := r.DS.ListMDMAndroidProfilesToSend(ctx, cursor, batchSize)
if err != nil {
return ctxerr.Wrap(ctx, err, "identify android profiles to send")
return 0, ctxerr.Wrap(ctx, err, "identify android profiles to send")
}
profilesByHostUUID := make(map[string][]*fleet.MDMAndroidProfilePayload)
@@ -108,7 +134,7 @@ func (r *profileReconciler) ReconcileProfiles(ctx context.Context) error {
profilesContents, err := r.DS.GetMDMAndroidProfilesContents(ctx, slices.Collect(maps.Keys(profilesToLoad)))
if err != nil {
return ctxerr.Wrap(ctx, err, "load android profiles content")
return 0, ctxerr.Wrap(ctx, err, "load android profiles content")
}
// index the to-remove profiles by host so we can pass them to sendHostProfiles
@@ -117,6 +143,21 @@ func (r *profileReconciler) ReconcileProfiles(ctx context.Context) error {
toRemoveByHostUUID[prof.HostUUID] = append(toRemoveByHostUUID[prof.HostUUID], prof)
}
// Collect all distinct host UUIDs for cursor advancement.
allHostUUIDs := make(map[string]struct{}, len(profilesByHostUUID)+len(toRemoveByHostUUID))
for uuid := range profilesByHostUUID {
allHostUUIDs[uuid] = struct{}{}
}
for uuid := range toRemoveByHostUUID {
allHostUUIDs[uuid] = struct{}{}
}
hostCount := len(allHostUUIDs)
// Track the last (lexicographically greatest) host UUID for cursor.
if hostCount > 0 {
r.lastHostUUID = slices.Max(slices.Collect(maps.Keys(allHostUUIDs)))
}
// Extract ONC cert aliases once for all hosts (profile contents are shared),
// then batch-fetch cert statuses for all hosts in a single DB query.
certAliases := extractProfileCertAliases(ctx, r.Logger, profilesContents)
@@ -124,7 +165,7 @@ func (r *profileReconciler) ReconcileProfiles(ctx context.Context) error {
if len(certAliases) > 0 {
allCertStatuses, err = r.DS.GetCertificateTemplateStatusesByNameForHosts(ctx, slices.Collect(maps.Keys(profilesByHostUUID)))
if err != nil {
return ctxerr.Wrap(ctx, err, "batch get certificate template statuses")
return 0, ctxerr.Wrap(ctx, err, "batch get certificate template statuses")
}
}
@@ -133,7 +174,7 @@ func (r *profileReconciler) ReconcileProfiles(ctx context.Context) error {
toRemove := toRemoveByHostUUID[hostUUID]
bulkProfs, err := r.sendHostProfiles(ctx, hostUUID, toInstallProfs, toRemove, profilesContents, certAliases, allCertStatuses[hostUUID])
if err != nil {
return ctxerr.Wrapf(ctx, err, "send profiles for host %s", hostUUID)
return 0, ctxerr.Wrapf(ctx, err, "send profiles for host %s", hostUUID)
}
bulkHostProfs = append(bulkHostProfs, bulkProfs...)
delete(toRemoveByHostUUID, hostUUID)
@@ -143,15 +184,15 @@ func (r *profileReconciler) ReconcileProfiles(ctx context.Context) error {
for hostUUID, toRemove := range toRemoveByHostUUID {
bulkProfs, err := r.sendHostProfiles(ctx, hostUUID, nil, toRemove, nil, nil, nil)
if err != nil {
return ctxerr.Wrapf(ctx, err, "send profiles for host %s", hostUUID)
return 0, ctxerr.Wrapf(ctx, err, "send profiles for host %s", hostUUID)
}
bulkHostProfs = append(bulkHostProfs, bulkProfs...)
}
if err := r.DS.BulkUpsertMDMAndroidHostProfiles(ctx, bulkHostProfs); err != nil {
return ctxerr.Wrap(ctx, err, "bulk upsert android host profiles")
return 0, ctxerr.Wrap(ctx, err, "bulk upsert android host profiles")
}
return nil
return hostCount, nil
}
func (r *profileReconciler) sendHostProfiles(
+33 -33
View File
@@ -122,13 +122,13 @@ func testNoHost(t *testing.T, ds fleet.Datastore, client *mock.Client, reconcile
}
// no host, so no calls to the Android API
err := reconciler.ReconcileProfiles(ctx)
_, err := reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
// run again, still nothing
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
@@ -149,13 +149,13 @@ func testHostsWithoutProfile(t *testing.T, ds fleet.Datastore, client *mock.Clie
createAndroidHost(t, ds, 2)
// nothing to process, no profiles missing nor extraneous
err := reconciler.ReconcileProfiles(ctx)
_, err := reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
// run again, still nothing to process
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
@@ -183,7 +183,7 @@ func testHostsWithProfile(t *testing.T, ds fleet.Datastore, client *mock.Client,
require.NoError(t, err)
// profile gets delivered to both hosts
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.True(t, client.EnterprisesPoliciesPatchFuncInvoked)
client.EnterprisesPoliciesPatchFuncInvoked = false
@@ -196,7 +196,7 @@ func testHostsWithProfile(t *testing.T, ds fleet.Datastore, client *mock.Client,
})
// run again, nothing to process as everything is pending
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
@@ -228,7 +228,7 @@ func testHostsWithConflictProfile(t *testing.T, ds fleet.Datastore, client *mock
require.NoError(t, err)
// profiles get delivered to both hosts, but p1 is failed
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.True(t, client.EnterprisesPoliciesPatchFuncInvoked)
client.EnterprisesPoliciesPatchFuncInvoked = false
@@ -243,7 +243,7 @@ func testHostsWithConflictProfile(t *testing.T, ds fleet.Datastore, client *mock
})
// run again, nothing to process as everything is pending/failed
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
@@ -283,7 +283,7 @@ func testHostsWithMultiOverrideProfile(t *testing.T, ds fleet.Datastore, client
require.NoError(t, err)
// profiles get delivered to h1 only
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.True(t, client.EnterprisesPoliciesPatchFuncInvoked)
client.EnterprisesPoliciesPatchFuncInvoked = false
@@ -297,7 +297,7 @@ func testHostsWithMultiOverrideProfile(t *testing.T, ds fleet.Datastore, client
})
// run again, nothing to process as everything is pending/failed
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
@@ -324,7 +324,7 @@ func testHostsWithAPIFailures(t *testing.T, ds fleet.Datastore, client *mock.Cli
require.NoError(t, err)
for i := range 3 {
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.True(t, client.EnterprisesPoliciesPatchFuncInvoked)
client.EnterprisesPoliciesPatchFuncInvoked = false
@@ -338,7 +338,7 @@ func testHostsWithAPIFailures(t *testing.T, ds fleet.Datastore, client *mock.Cli
}
// next run marks as failed
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
@@ -349,7 +349,7 @@ func testHostsWithAPIFailures(t *testing.T, ds fleet.Datastore, client *mock.Cli
})
// next run has nothing to do
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
@@ -365,7 +365,7 @@ func testHostsWithAPIFailures(t *testing.T, ds fleet.Datastore, client *mock.Cli
return policy, nil
}
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.True(t, client.EnterprisesPoliciesPatchFuncInvoked)
client.EnterprisesPoliciesPatchFuncInvoked = false
@@ -404,7 +404,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie
p1Checksum := getAndroidProfileChecksum(t, ds, p1.ProfileUUID)
// profiles get delivered
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.True(t, client.EnterprisesPoliciesPatchFuncInvoked)
client.EnterprisesPoliciesPatchFuncInvoked = false
@@ -417,7 +417,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie
})
// run again, nothing to process
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
@@ -431,7 +431,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie
require.NoError(t, err)
// run again, nothing to process
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
@@ -452,7 +452,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie
require.NoError(t, err)
// profile gets re-delivered
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.True(t, client.EnterprisesPoliciesPatchFuncInvoked)
client.EnterprisesPoliciesPatchFuncInvoked = false
@@ -471,7 +471,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie
p2Checksum := getAndroidProfileChecksum(t, ds, p2.ProfileUUID)
// profiles get re-delivered
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.True(t, client.EnterprisesPoliciesPatchFuncInvoked)
client.EnterprisesPoliciesPatchFuncInvoked = false
@@ -486,7 +486,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie
})
// run again, nothing to process
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
@@ -500,7 +500,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie
require.NoError(t, err)
// run again, nothing to process
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
@@ -516,7 +516,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie
}
// profiles get re-delivered
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.True(t, client.EnterprisesPoliciesPatchFuncInvoked)
client.EnterprisesPoliciesPatchFuncInvoked = false
@@ -532,7 +532,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie
})
// run again, nothing to process
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
@@ -552,7 +552,7 @@ func testHostsWithAddRemoveUpdateProfiles(t *testing.T, ds fleet.Datastore, clie
})
// run again, nothing to process
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
@@ -609,7 +609,7 @@ func testHostsWithLabelProfiles(t *testing.T, ds fleet.Datastore, client *mock.C
}
// currently only the no-label profile is applied
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.True(t, client.EnterprisesPoliciesPatchFuncInvoked)
client.EnterprisesPoliciesPatchFuncInvoked = false
@@ -631,7 +631,7 @@ func testHostsWithLabelProfiles(t *testing.T, ds fleet.Datastore, client *mock.C
// no-label and exclude any are applied
version++
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.True(t, client.EnterprisesPoliciesPatchFuncInvoked)
client.EnterprisesPoliciesPatchFuncInvoked = false
@@ -653,7 +653,7 @@ func testHostsWithLabelProfiles(t *testing.T, ds fleet.Datastore, client *mock.C
// no-label, exclude any and the respective include profiles are applied
version++
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.True(t, client.EnterprisesPoliciesPatchFuncInvoked)
client.EnterprisesPoliciesPatchFuncInvoked = false
@@ -676,7 +676,7 @@ func testHostsWithLabelProfiles(t *testing.T, ds fleet.Datastore, client *mock.C
// this only affects h1, h2 version is unchanged
h2Version := version
version++
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.True(t, client.EnterprisesPoliciesPatchFuncInvoked)
client.EnterprisesPoliciesPatchFuncInvoked = false
@@ -694,7 +694,7 @@ func testHostsWithLabelProfiles(t *testing.T, ds fleet.Datastore, client *mock.C
})
// run again, nothing to process
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
require.False(t, client.EnterprisesPoliciesPatchFuncInvoked)
require.False(t, client.EnterprisesDevicesPatchFuncInvoked)
@@ -1314,7 +1314,7 @@ func testONCWithheldUntilCertVerified(t *testing.T, ds fleet.Datastore, client *
require.NoError(t, err)
// --- Phase 1: cert is pending, ONC should be withheld, non-ONC applied ---
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
assertHostProfiles(t, ds, []*fleet.MDMAndroidProfilePayload{
@@ -1346,7 +1346,7 @@ func testONCWithheldUntilCertVerified(t *testing.T, ds fleet.Datastore, client *
client.EnterprisesPoliciesPatchFuncInvoked = false
client.EnterprisesDevicesPatchFuncInvoked = false
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
// Both profiles should now be applied (included in policy, with request UUIDs)
@@ -1383,7 +1383,7 @@ func testONCWithheldUntilCertVerified(t *testing.T, ds fleet.Datastore, client *
})
// cert is "delivered" (not terminal), ONC should be withheld
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
phase3Profiles, err := ds.GetHostMDMAndroidProfiles(ctx, host.UUID)
@@ -1403,7 +1403,7 @@ func testONCWithheldUntilCertVerified(t *testing.T, ds fleet.Datastore, client *
)
return err
})
err = reconciler.ReconcileProfiles(ctx)
_, err = reconciler.ReconcileProfiles(ctx, "", 0)
require.NoError(t, err)
// Both profiles applied (cert is terminally failed, ONC released)
+27 -3
View File
@@ -1888,7 +1888,11 @@ type GetHostMDMAndroidProfilesFunc func(ctx context.Context, hostUUID string) ([
type NewAndroidPolicyRequestFunc func(ctx context.Context, req *android.MDMAndroidPolicyRequest) error
type ListMDMAndroidProfilesToSendFunc func(ctx context.Context) ([]*fleet.MDMAndroidProfilePayload, []*fleet.MDMAndroidProfilePayload, error)
type ListMDMAndroidProfilesToSendFunc func(ctx context.Context, cursor string, batchSize int) ([]*fleet.MDMAndroidProfilePayload, []*fleet.MDMAndroidProfilePayload, error)
type GetMDMAndroidReconcileCursorFunc func(ctx context.Context) (string, error)
type SetMDMAndroidReconcileCursorFunc func(ctx context.Context, cursor string) error
type GetMDMAndroidProfilesContentsFunc func(ctx context.Context, uuids []string) (map[string]json.RawMessage, error)
@@ -4900,6 +4904,12 @@ type DataStore struct {
ListMDMAndroidProfilesToSendFunc ListMDMAndroidProfilesToSendFunc
ListMDMAndroidProfilesToSendFuncInvoked bool
GetMDMAndroidReconcileCursorFunc GetMDMAndroidReconcileCursorFunc
GetMDMAndroidReconcileCursorFuncInvoked bool
SetMDMAndroidReconcileCursorFunc SetMDMAndroidReconcileCursorFunc
SetMDMAndroidReconcileCursorFuncInvoked bool
GetMDMAndroidProfilesContentsFunc GetMDMAndroidProfilesContentsFunc
GetMDMAndroidProfilesContentsFuncInvoked bool
@@ -11742,11 +11752,25 @@ func (s *DataStore) NewAndroidPolicyRequest(ctx context.Context, req *android.MD
return s.NewAndroidPolicyRequestFunc(ctx, req)
}
func (s *DataStore) ListMDMAndroidProfilesToSend(ctx context.Context) ([]*fleet.MDMAndroidProfilePayload, []*fleet.MDMAndroidProfilePayload, error) {
func (s *DataStore) ListMDMAndroidProfilesToSend(ctx context.Context, cursor string, batchSize int) ([]*fleet.MDMAndroidProfilePayload, []*fleet.MDMAndroidProfilePayload, error) {
s.mu.Lock()
s.ListMDMAndroidProfilesToSendFuncInvoked = true
s.mu.Unlock()
return s.ListMDMAndroidProfilesToSendFunc(ctx)
return s.ListMDMAndroidProfilesToSendFunc(ctx, cursor, batchSize)
}
func (s *DataStore) GetMDMAndroidReconcileCursor(ctx context.Context) (string, error) {
s.mu.Lock()
s.GetMDMAndroidReconcileCursorFuncInvoked = true
s.mu.Unlock()
return s.GetMDMAndroidReconcileCursorFunc(ctx)
}
func (s *DataStore) SetMDMAndroidReconcileCursor(ctx context.Context, cursor string) error {
s.mu.Lock()
s.SetMDMAndroidReconcileCursorFuncInvoked = true
s.mu.Unlock()
return s.SetMDMAndroidReconcileCursorFunc(ctx, cursor)
}
func (s *DataStore) GetMDMAndroidProfilesContents(ctx context.Context, uuids []string) (map[string]json.RawMessage, error) {
+1 -1
View File
@@ -485,7 +485,7 @@ func (s *integrationMDMTestSuite) SetupSuite() {
err := android_service.ReconcileProfilesWithClient(ctx, ds, logger, "", androidMockClient, config.AndroidAgentConfig{
Package: "com.fleetdm.agent",
SigningSHA256: "abc123def456",
})
}, 0)
require.NoError(s.T(), err)
return err
}),