Update server-proto version to 9, implement THROTTLED w/ 24h cooldown (#38920)

<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #37072 

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

- [x] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements)
- [x] If paths of existing endpoints are modified without backwards
compatibility, checked the frontend/CLI for any necessary changes

## Testing

- [x] Added/updated automated tests
- [x] Where appropriate, [automated tests simulate multiple hosts and
test for host
isolation](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/reference/patterns-backend.md#unit-testing)
(updates to one hosts's records do not affect another)

- [x] QA'd all new/changed functionality manually
This commit is contained in:
Jordan Montgomery
2026-01-29 15:31:28 -05:00
committed by GitHub
parent 46df5803e8
commit a1e5c500c0
12 changed files with 241 additions and 16 deletions
+1
View File
@@ -0,0 +1 @@
* Updated DEP syncing code to use server-protocol-version 9 and handle THROTTLED responses
+21 -6
View File
@@ -4633,6 +4633,7 @@ func (ds *Datastore) updateHostDEPAssignProfileResponses(ctx context.Context, pa
var (
notAccessible []string
failed []string
throttled []string
)
for serial, status := range payload.Devices {
@@ -4643,6 +4644,8 @@ func (ds *Datastore) updateHostDEPAssignProfileResponses(ctx context.Context, pa
notAccessible = append(notAccessible, serial)
case string(fleet.DEPAssignProfileResponseFailed):
failed = append(failed, serial)
case string(fleet.DEPAssignProfileResponseThrottled):
throttled = append(throttled, serial)
default:
// this should never happen unless Apple changes the response format, so we log it for
// future debugging
@@ -4663,6 +4666,10 @@ func (ds *Datastore) updateHostDEPAssignProfileResponses(ctx context.Context, pa
string(fleet.DEPAssignProfileResponseFailed), abmTokenID); err != nil {
return err
}
if err := updateHostDEPAssignProfileResponses(ctx, tx, ds.logger, payload.ProfileUUID, throttled,
string(fleet.DEPAssignProfileResponseThrottled), abmTokenID); err != nil {
return err
}
return nil
})
}
@@ -4714,8 +4721,11 @@ WHERE
return nil
}
// depCooldownPeriod is the waiting period following a failed DEP assign profile request for a host.
const depCooldownPeriod = 1 * time.Hour // TODO: Make this a test config option?
// depFailedCooldownPeriod is the waiting period following a failed DEP assign profile request for a host.
const (
depFailedCooldownPeriod = 1 * time.Hour // TODO: Make this a test config option?
depThrottledCooldownPeriod = 24 * time.Hour
)
func (ds *Datastore) ScreenDEPAssignProfileSerialsForCooldown(ctx context.Context, serials []string) (skipSerialsByOrgName map[string][]string, serialsByOrgName map[string][]string, err error) {
if len(serials) == 0 {
@@ -4724,7 +4734,8 @@ func (ds *Datastore) ScreenDEPAssignProfileSerialsForCooldown(ctx context.Contex
stmt := `
SELECT
CASE WHEN assign_profile_response = ? AND (response_updated_at > DATE_SUB(NOW(), INTERVAL ? SECOND) OR retry_job_id != 0) THEN
CASE WHEN (assign_profile_response = ? AND (response_updated_at > DATE_SUB(NOW(), INTERVAL ? SECOND) OR retry_job_id != 0)) OR
(assign_profile_response = ? AND (response_updated_at > DATE_SUB(NOW(), INTERVAL ? SECOND) OR retry_job_id != 0)) THEN
'skip'
ELSE
'assign'
@@ -4739,7 +4750,7 @@ WHERE
h.hardware_serial IN (?)
`
stmt, args, err := sqlx.In(stmt, string(fleet.DEPAssignProfileResponseFailed), depCooldownPeriod.Seconds(), serials)
stmt, args, err := sqlx.In(stmt, string(fleet.DEPAssignProfileResponseFailed), depFailedCooldownPeriod.Seconds(), string(fleet.DEPAssignProfileResponseThrottled), depThrottledCooldownPeriod.Seconds(), serials)
if err != nil {
return nil, nil, ctxerr.Wrap(ctx, err, "screen dep serials: prepare statement arguments")
}
@@ -4791,10 +4802,14 @@ FROM
JOIN hosts h ON h.id = host_id
LEFT JOIN jobs j ON j.id = retry_job_id
WHERE
assign_profile_response = ?
(assign_profile_response = ?
AND(retry_job_id = 0 OR j.state = ?)
AND(response_updated_at IS NULL
OR response_updated_at <= DATE_SUB(NOW(), INTERVAL ? SECOND))
) OR (assign_profile_response = ?
AND(retry_job_id = 0 OR j.state = ?)
AND(response_updated_at IS NULL
OR response_updated_at <= DATE_SUB(NOW(), INTERVAL ? SECOND)))
ORDER BY response_updated_at ASC
LIMIT ?`
@@ -4802,7 +4817,7 @@ LIMIT ?`
TeamID uint `db:"team_id"`
HardwareSerial string `db:"hardware_serial"`
}
if err := sqlx.SelectContext(ctx, ds.reader(ctx), &rows, stmt, string(fleet.DEPAssignProfileResponseFailed), string(fleet.JobStateFailure), depCooldownPeriod.Seconds(), apple_mdm.DEPSyncLimit); err != nil {
if err := sqlx.SelectContext(ctx, ds.reader(ctx), &rows, stmt, string(fleet.DEPAssignProfileResponseFailed), string(fleet.JobStateFailure), depFailedCooldownPeriod.Seconds(), string(fleet.DEPAssignProfileResponseThrottled), string(fleet.JobStateFailure), depThrottledCooldownPeriod.Seconds(), apple_mdm.DEPSyncLimit); err != nil {
return nil, ctxerr.Wrap(ctx, err, "get host dep assign profile expired cooldowns")
}
+2 -2
View File
@@ -9714,7 +9714,7 @@ func testGetDEPAssignProfileExpiredCooldowns(t *testing.T, ds *Datastore) {
// Set response_updated_at to be dep cooldown + 10 seconds to avoid timing issues
ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
_, err := q.ExecContext(ctx, `UPDATE host_dep_assignments SET response_updated_at = DATE_SUB(NOW(), INTERVAL ? SECOND) WHERE host_id = ?`, depCooldownPeriod.Seconds()+10, host.ID)
_, err := q.ExecContext(ctx, `UPDATE host_dep_assignments SET response_updated_at = DATE_SUB(NOW(), INTERVAL ? SECOND) WHERE host_id = ?`, depFailedCooldownPeriod.Seconds()+10, host.ID)
return err
})
cooldowns, err = ds.GetDEPAssignProfileExpiredCooldowns(ctx)
@@ -9725,7 +9725,7 @@ func testGetDEPAssignProfileExpiredCooldowns(t *testing.T, ds *Datastore) {
for i := range 200 {
h := newTestHostWithPlatform(t, ds, fmt.Sprintf("host-%d", i), "macos", nil)
ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
_, err := q.ExecContext(ctx, `INSERT INTO host_dep_assignments (host_id, profile_uuid, assign_profile_response, response_updated_at, retry_job_id) VALUES (?, ?, ?, DATE_SUB(NOW(), INTERVAL ? SECOND), 0)`, h.ID, uuid.NewString(), fleet.DEPAssignProfileResponseFailed, depCooldownPeriod.Seconds()+10)
_, err := q.ExecContext(ctx, `INSERT INTO host_dep_assignments (host_id, profile_uuid, assign_profile_response, response_updated_at, retry_job_id) VALUES (?, ?, ?, DATE_SUB(NOW(), INTERVAL ? SECOND), 0)`, h.ID, uuid.NewString(), fleet.DEPAssignProfileResponseFailed, depFailedCooldownPeriod.Seconds()+10)
return err
})
}
+1 -1
View File
@@ -927,7 +927,7 @@ const hostMDMSelect = `,
'enrollment_status', hmdm.enrollment_status,
'dep_profile_error',
CASE
WHEN hdep.assign_profile_response = '` + string(fleet.DEPAssignProfileResponseFailed) + `' THEN CAST(TRUE AS JSON)
WHEN hdep.assign_profile_response IN ('` + string(fleet.DEPAssignProfileResponseFailed) + `', '` + string(fleet.DEPAssignProfileResponseThrottled) + `') THEN CAST(TRUE AS JSON)
ELSE CAST(FALSE AS JSON)
END,
'server_url',
+6 -3
View File
@@ -554,6 +554,7 @@ const (
DEPAssignProfileResponseSuccess DEPAssignProfileResponseStatus = "SUCCESS"
DEPAssignProfileResponseNotAccessible DEPAssignProfileResponseStatus = "NOT_ACCESSIBLE"
DEPAssignProfileResponseFailed DEPAssignProfileResponseStatus = "FAILED"
DEPAssignProfileResponseThrottled DEPAssignProfileResponseStatus = "THROTTLED"
)
// NanoEnrollment represents a row in the nano_enrollments table managed by
@@ -1118,9 +1119,11 @@ type AppleMDMVPPInstaller interface {
InstallVPPAppPostValidation(ctx context.Context, host *Host, vppApp *VPPApp, token string, opts HostSoftwareInstallOptions) (string, error)
}
const DeviceLocationCmdName = "DeviceLocation"
const EnableLostModeCmdName = "EnableLostMode"
const DisableLostModeCmdName = "DisableLostMode"
const (
DeviceLocationCmdName = "DeviceLocation"
EnableLostModeCmdName = "EnableLostMode"
DisableLostModeCmdName = "DisableLostMode"
)
type HostLocationData struct {
HostID uint `db:"host_id"`
+9
View File
@@ -297,6 +297,15 @@ func TestIsEligibleForDEPMigration(t *testing.T) {
expected: false,
expectedManual: false,
},
{
name: "Not eligible - DEP assigned and DEP profile throttled",
osqueryHostID: ptr.String("some-id"),
depAssignedToFleet: ptr.Bool(true),
depProfileResponse: DEPAssignProfileResponseThrottled,
enrolledInThirdPartyMDM: true,
expected: false,
expectedManual: false,
},
{
name: "Not eligible - DEP assigned but not response yet",
osqueryHostID: ptr.String("some-id"),
+1 -1
View File
@@ -950,7 +950,7 @@ func (d *DEPService) getProfileUUIDForTeam(ctx context.Context, tmID *uint, abmT
// logCountsForResults tries to aggregate the result types and log the counts.
func logCountsForResults(deviceResults map[string]string) (out []interface{}) {
results := map[string]int{"success": 0, "not_accessible": 0, "failed": 0, "other": 0}
results := map[string]int{"success": 0, "not_accessible": 0, "failed": 0, "throttled": 0, "other": 0}
for _, result := range deviceResults {
l := strings.ToLower(result)
if _, ok := results[l]; !ok {
+100
View File
@@ -428,4 +428,104 @@ func TestDEPService_RunAssigner(t *testing.T) {
return nil
})
})
t.Run("assign returns throttled for one device", func(t *testing.T) {
start := time.Now().Truncate(time.Second)
devices := []godep.Device{
{SerialNumber: "a", OpType: "added"},
{SerialNumber: "b", OpType: "ignore"},
{SerialNumber: "c", OpType: ""},
}
var assignCalled bool
svc := setupTest(t, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
encoder := json.NewEncoder(w)
switch r.URL.Path {
case "/session":
_, _ = w.Write([]byte(`{"auth_session_token": "session123"}`))
case "/account":
_, _ = w.Write(fmt.Appendf(nil, `{"admin_id": "admin123", "org_name": "%s"}`, abmTokenOrgName))
case "/profile":
err := encoder.Encode(godep.ProfileResponse{ProfileUUID: "profile123"})
require.NoError(t, err)
case "/server/devices":
err := encoder.Encode(godep.DeviceResponse{})
require.NoError(t, err)
case "/devices/sync":
err := encoder.Encode(godep.DeviceResponse{Devices: devices})
require.NoError(t, err)
case "/profile/devices":
assignCalled = true
reqBody, err := io.ReadAll(r.Body)
require.NoError(t, err)
var assignReq godep.Profile
err = json.Unmarshal(reqBody, &assignReq)
require.NoError(t, err)
require.Equal(t, assignReq.ProfileUUID, "profile123")
require.ElementsMatch(t, []string{"a", "c"}, assignReq.Devices)
apiResp := godep.ProfileResponse{
ProfileUUID: "profile123",
Devices: map[string]string{
"a": string(fleet.DEPAssignProfileResponseSuccess),
"c": string(fleet.DEPAssignProfileResponseThrottled),
},
}
respBytes, err := json.Marshal(&apiResp)
require.NoError(t, err)
_, err = w.Write(respBytes)
require.NoError(t, err)
default:
t.Errorf("unexpected request to %s", r.URL.Path)
}
})
err := svc.RunAssigner(ctx)
require.NoError(t, err)
require.True(t, assignCalled)
// the default profile was created
defProf, err := ds.GetMDMAppleEnrollmentProfileByType(ctx, fleet.MDMAppleEnrollmentTypeAutomatic)
require.NoError(t, err)
require.NotNil(t, defProf)
require.NotEmpty(t, defProf.Token)
// a profile UUID was assigned to no-team
profUUID, modTime, err := ds.GetMDMAppleDefaultSetupAssistant(ctx, nil, abmTokenOrgName)
require.NoError(t, err)
require.Equal(t, "profile123", profUUID)
require.False(t, modTime.Before(start))
// a couple hosts were created (except the op_type ignored)
hosts, err := ds.ListHosts(ctx, fleet.TeamFilter{User: test.UserAdmin}, fleet.HostListOptions{})
require.NoError(t, err)
require.Len(t, hosts, 2)
serials := make([]string, len(hosts))
for i, h := range hosts {
serials[i] = h.HardwareSerial
require.Nil(t, h.TeamID, h.HardwareSerial)
}
require.ElementsMatch(t, []string{"a", "c"}, serials)
// Verify that the one host has an assignment marked throttled
mysql.ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
stmt := "SELECT COUNT(*) FROM host_dep_assignments WHERE host_id = ? AND assign_profile_response = ?"
var result int
require.NoError(t, sqlx.GetContext(ctx, q, &result, stmt, hosts[1].ID, fleet.DEPAssignProfileResponseThrottled))
require.Equal(t, 1, result, "expected one throttled assignment for serial c")
return nil
})
// And the other is marked success
mysql.ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
stmt := "SELECT COUNT(*) FROM host_dep_assignments WHERE host_id = ? AND assign_profile_response = ?"
var result int
require.NoError(t, sqlx.GetContext(ctx, q, &result, stmt, hosts[0].ID, fleet.DEPAssignProfileResponseSuccess))
require.Equal(t, 1, result, "expected one success assignment for serial a")
return nil
})
})
}
+1 -1
View File
@@ -16,7 +16,7 @@ const (
ADMAuthSession = "X-ADM-Auth-Session"
ServerProtocolVersion = "X-Server-Protocol-Version"
DefaultServerProtocolVersion = "8"
DefaultServerProtocolVersion = "9"
SessionEndpoint = "/session"
+19
View File
@@ -417,6 +417,25 @@ func TestGetFleetDesktopSummary(t *testing.T) {
RenewEnrollmentProfile: false,
},
},
{
name: "throttled ADE assignment status",
host: &fleet.Host{
DEPAssignedToFleet: ptr.Bool(true),
OsqueryHostID: ptr.String("test"),
},
hostMDM: &fleet.HostMDM{
IsServer: false,
InstalledFromDep: true,
Enrolled: true,
Name: fleet.WellKnownMDMIntune,
DEPProfileAssignStatus: ptr.String(string(fleet.DEPAssignProfileResponseThrottled)),
},
err: nil,
out: fleet.DesktopNotifications{
NeedsMDMMigration: false,
RenewEnrollmentProfile: false,
},
},
{
name: "not accessible ADE assignment status",
host: &fleet.Host{
+64 -2
View File
@@ -1014,6 +1014,7 @@ func (s *integrationMDMTestSuite) TestDEPProfileAssignment() {
}
expectAssignProfileResponseFailed := "" // set to device serial when testing the failed profile assignment flow
expectAssignProfileResponseThrottled := "" // set to device serial when testing the throttled profile assignment flow
expectAssignProfileResponseNotAccessible := "" // set to device serial when testing the not accessible profile assignment flow
s.enableABM(t.Name())
@@ -1052,6 +1053,8 @@ func (s *integrationMDMTestSuite) TestDEPProfileAssignment() {
resp.Devices[device] = string(fleet.DEPAssignProfileResponseNotAccessible)
case expectAssignProfileResponseFailed:
resp.Devices[device] = string(fleet.DEPAssignProfileResponseFailed)
case expectAssignProfileResponseThrottled:
resp.Devices[device] = string(fleet.DEPAssignProfileResponseThrottled)
default:
resp.Devices[device] = string(fleet.DEPAssignProfileResponseSuccess)
}
@@ -1651,6 +1654,66 @@ func (s *integrationMDMTestSuite) TestDEPProfileAssignment() {
checkHostCooldown(serial, expectProfileUUID, fleet.DEPAssignProfileResponseNotAccessible, &failedAt, expectNoJobID) // no change
checkNoJobsPending()
expectAssignProfileResponseNotAccessible = ""
// ingest new device via DEP but the profile assignment fails
serial = uuid.NewString()
devices = []godep.Device{
{SerialNumber: serial, Model: "MacBook Pro", OS: "osx", OpType: "added"},
}
expectAssignProfileResponseThrottled = serial
profileAssignmentReqs = []profileAssignmentReq{}
s.runDEPSchedule()
checkAssignProfileRequests(serial, nil)
profUUID = profileAssignmentReqs[0].ProfileUUID
d = checkHostCooldown(serial, profUUID, fleet.DEPAssignProfileResponseThrottled, nil, expectNoJobID)
require.NotZero(t, d.ResponseUpdatedAt)
failedAt = d.ResponseUpdatedAt
checkNoJobsPending()
h = checkListHostDEPError(serial, "Pending", true) // list hosts shows device pending and dep profile error
// transfer to team, no profile assignment request is made during the cooldown period
profileAssignmentReqs = []profileAssignmentReq{}
s.Do("POST", "/api/v1/fleet/hosts/transfer",
addHostsToTeamRequest{TeamID: &team.ID, HostIDs: []uint{h.ID}}, http.StatusOK)
checkPendingMacOSSetupAssistantJob("hosts_transferred", &team.ID, []string{serial}, 0)
s.runIntegrationsSchedule()
require.Empty(t, profileAssignmentReqs) // screened by cooldown
checkHostCooldown(serial, profUUID, fleet.DEPAssignProfileResponseThrottled, &failedAt, expectNoJobID) // no change
checkNoJobsPending()
// run the integrations schedule and expect no changes
profileAssignmentReqs = []profileAssignmentReq{}
s.runIntegrationsSchedule()
require.Empty(t, profileAssignmentReqs)
checkHostCooldown(serial, profUUID, fleet.DEPAssignProfileResponseThrottled, &failedAt, expectNoJobID) // no change
checkNoJobsPending()
// simulate expired cooldown
failedAt = failedAt.Add(-25 * time.Hour)
setAssignProfileResponseUpdatedAt(serial, failedAt)
profileAssignmentReqs = []profileAssignmentReq{}
s.runIntegrationsSchedule()
require.Empty(t, profileAssignmentReqs) // assign profile request will be made when the retry job is processed on the next worker run
d = checkHostCooldown(serial, profUUID, fleet.DEPAssignProfileResponseThrottled, &failedAt, nil)
require.NotZero(t, d.RetryJobID) // retry job created
jobID = d.RetryJobID
checkPendingMacOSSetupAssistantJob("hosts_cooldown", &team.ID, []string{serial}, jobID)
// run the inregration schedule and expect success
expectAssignProfileResponseThrottled = ""
profileAssignmentReqs = []profileAssignmentReq{}
s.runIntegrationsSchedule()
checkAssignProfileRequests(serial, nil)
require.NotEqual(t, profUUID, profileAssignmentReqs[0].ProfileUUID) // retry job will use the current team profile instead
profUUID = profileAssignmentReqs[0].ProfileUUID
d = checkHostCooldown(serial, profUUID, fleet.DEPAssignProfileResponseSuccess, nil, expectNoJobID) // retry job cleared
require.True(t, d.ResponseUpdatedAt.After(failedAt))
checkNoJobsPending()
// list hosts shows pending (because MDM detail query hasn't been reported) but dep profile
// error has been cleared
checkListHostDEPError(serial, "Pending", false)
// run with devices that already have valid and invalid profiles
// assigned, we shouldn't re-assign the valid ones.
devices = []godep.Device{
@@ -1659,9 +1722,8 @@ func (s *integrationMDMTestSuite) TestDEPProfileAssignment() {
{SerialNumber: uuid.NewString(), Model: "MacBook Pro", OS: "osx", OpType: "added", ProfileUUID: "bar"}, // doesn't match an existing profile
{SerialNumber: uuid.NewString(), Model: "MacBook Mini", OS: "osx", OpType: "modified", ProfileUUID: "foo"}, // doesn't match an existing profile
{SerialNumber: addedSerial, Model: "MacBook Pro", OS: "osx", OpType: "added", ProfileUUID: defaultProfileUUID}, // matches existing profile, but will be assigned since it is "added"
{SerialNumber: serial, Model: "MacBook Mini", OS: "osx", OpType: "modified", ProfileUUID: defaultProfileUUID}, // matches existing profile
{SerialNumber: serial, Model: "MacBook Mini", OS: "osx", OpType: "modified", ProfileUUID: profUUID}, // matches existing profile
}
expectAssignProfileResponseNotAccessible = ""
profileAssignmentReqs = []profileAssignmentReq{}
s.runDEPSchedule()
require.NotEmpty(t, profileAssignmentReqs)
+16
View File
@@ -7041,6 +7041,22 @@ func (s *integrationMDMTestSuite) TestMDMMigration() {
require.NoError(t, s.ds.DeleteHostDEPAssignments(ctx, abmToken.ID, []string{host.HardwareSerial}))
cleanAssignmentStatus()
// simulate a "THROTTLED" JSON profile assignment
profileAssignmentStatusResponse = fleet.DEPAssignProfileResponseThrottled
s.runDEPSchedule()
getDesktopResp = fleetDesktopResponse{}
res = s.DoRawNoAuth("GET", "/api/latest/fleet/device/"+token+"/desktop", nil, http.StatusOK)
require.NoError(t, json.NewDecoder(res.Body).Decode(&getDesktopResp))
require.NoError(t, res.Body.Close())
require.False(t, getDesktopResp.Notifications.NeedsMDMMigration)
require.False(t, orbitConfigResp.Notifications.RenewEnrollmentProfile)
orbitConfigResp = orbitGetConfigResponse{}
s.DoJSON("POST", "/api/fleet/orbit/config", json.RawMessage(fmt.Sprintf(`{"orbit_node_key": %q}`, *host.OrbitNodeKey)), http.StatusOK, &orbitConfigResp)
require.False(t, orbitConfigResp.Notifications.NeedsMDMMigration)
require.False(t, orbitConfigResp.Notifications.RenewEnrollmentProfile)
require.NoError(t, s.ds.DeleteHostDEPAssignments(ctx, abmToken.ID, []string{host.HardwareSerial}))
cleanAssignmentStatus()
// simulate a "NOT_ACCESSIBLE" JSON profile assignment
profileAssignmentStatusResponse = fleet.DEPAssignProfileResponseNotAccessible
s.runDEPSchedule()