Files
Nico a335b3e6d4 Fix VPP API retry recursion causing server OOM (#46659)
<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #46656

`server/mdm/apple/vpp.do` retried transient Apple errors by **calling
itself recursively**, with the rate-limit branch nesting `retry.Do`
inside `retry.Do`.

This change replaces the recursion with a single retry loop (respecting
the prior 1 initial attempt + 3 retries), closes each response before
retrying, honors Apple's `Retry-After` capped at 30s so that a
multi-minute value can't block a synchronous request, and threads
`context` through the VPP calls so the backoff is cancellable. The retry
timings are otherwise unchanged from before.

Following @sgress454 suggestion, I considered routing this through the
shared `retry.Do` helper (a single attempt wrapped in `retry.Do` + an
error filter) but figured out that:
- retry.Do` owns its own wait schedule and its error filter returns an
outcome enum rather than a duration, so it can't honor Apple's
per-response `Retry-After` value.
- also, I'd have to change the `retry` package to receive an extra `ctx`
param so that the backoff is context-aware (which IMHO is more blast
radius than this incident fix should carry).

# Checklist for submitter

- [x] Changes file added for user-visible changes in `changes/`.
- [x] Timeouts are implemented and retries are limited to avoid infinite
loops

## Testing

- [x] Added/updated automated tests
- [x] QA'd all new/changed functionality manually

**What was verified.** The new automated test cannot run against `main`
(the fix changes the VPP function signatures and adds the retry knobs),
so to confirm the actual failure mode I checked out `main` and ran a
small repro that drives the VPP client against an Apple endpoint that
always returns the rate-limit error. On `main`, the call **never
returns** — `do()` recurses without bound — and the repro times out:

```
--- FAIL: TestReproUnboundedRecursionOnMain (10.00s)
    zz_repro_main_test.go:30: AssociateAssets did NOT return within 10s — unbounded retry recursion in do() on main
FAIL
FAIL	github.com/fleetdm/fleet/v4/server/mdm/apple/vpp	10.642s
```

On this branch the same scenario returns a bounded error promptly. That
behavior is covered by the new `TestDoRetryIsBoundedAndNonRecursive`
(bounded rate-limit retries, `Retry-After` honored-but-capped, and
context cancellation), and the full `server/mdm/apple/vpp` package
passes.
**I did not perform an end-to-end QA against a live Apple endpoint**.


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Bug Fixes**
* Fixed a server out-of-memory crash that occurred when Apple VPP API
repeatedly returned transient errors during VPP operations, including
app installs, user registration, and license seat releases.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
2026-06-02 16:18:02 -03:00

902 lines
29 KiB
Go

package vpp
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
"github.com/fleetdm/fleet/v4/server/dev_mode"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func setupFakeServer(t *testing.T, handler http.HandlerFunc) {
server := httptest.NewServer(handler)
dev_mode.SetOverride("FLEET_DEV_VPP_URL", server.URL, t)
t.Cleanup(server.Close)
}
func TestGetConfig(t *testing.T) {
tests := []struct {
name string
token string
handler http.HandlerFunc
wantName string
wantCountry string
expectedErrMsg string
expectMinCalls int
expectMaxCalls int
}{
{
name: "valid token US",
token: "valid_token",
handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
fmt.Fprintln(w, `{"locationName": "Test Location", "countryISO2ACode": "US"}`)
},
wantName: "Test Location",
wantCountry: "us",
expectedErrMsg: "",
expectMinCalls: 1,
expectMaxCalls: 1,
},
{
name: "valid token DE lowercased",
token: "valid_token",
handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
fmt.Fprintln(w, `{"locationName": "DE Org", "countryISO2ACode": "DE"}`)
},
wantName: "DE Org",
wantCountry: "de",
expectedErrMsg: "",
expectMinCalls: 1,
expectMaxCalls: 1,
},
{
name: "invalid token",
token: "invalid_token",
handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusUnauthorized)
fmt.Fprintln(w, `{"errorNumber": 9622}`)
},
wantName: "",
wantCountry: "",
expectedErrMsg: "making request to Apple VPP endpoint: Apple VPP endpoint returned error: (error number: 9622)",
// Apple application errors should not be retried.
expectMinCalls: 1,
expectMaxCalls: 1,
},
{
name: "server error retries up to 3 times",
token: "valid_token",
handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintln(w, `Internal Server Error`)
},
wantName: "",
wantCountry: "",
expectedErrMsg: "calling Apple VPP endpoint failed with status 500: Internal Server Error\n",
expectMinCalls: 3,
expectMaxCalls: 3,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var calls int
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
calls++
tt.handler(w, r)
})
cfg, err := GetConfig(t.Context(), tt.token)
if tt.expectedErrMsg != "" {
require.Error(t, err)
require.Contains(t, err.Error(), tt.expectedErrMsg)
} else {
require.NoError(t, err)
}
require.Equal(t, tt.wantName, cfg.LocationName)
require.Equal(t, tt.wantCountry, cfg.CountryCode)
if tt.expectMinCalls > 0 {
require.GreaterOrEqual(t, calls, tt.expectMinCalls)
require.LessOrEqual(t, calls, tt.expectMaxCalls)
}
})
}
t.Run("transient failure then success", func(t *testing.T) {
var calls int
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
calls++
if calls < 2 {
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintln(w, `Internal Server Error`)
return
}
w.WriteHeader(http.StatusOK)
fmt.Fprintln(w, `{"locationName": "Recovered", "countryISO2ACode": "FR"}`)
})
cfg, err := GetConfig(t.Context(), "token")
require.NoError(t, err)
require.Equal(t, "Recovered", cfg.LocationName)
require.Equal(t, "fr", cfg.CountryCode)
require.Equal(t, 2, calls)
})
}
func TestAssociateAssetsRequestValidate(t *testing.T) {
t.Run("serial numbers only is valid", func(t *testing.T) {
req := &AssociateAssetsRequest{SerialNumbers: []string{"SN1"}}
require.NoError(t, req.Validate())
})
t.Run("client user ids only is valid", func(t *testing.T) {
req := &AssociateAssetsRequest{ClientUserIds: []string{"user-1"}}
require.NoError(t, req.Validate())
})
t.Run("both populated is rejected", func(t *testing.T) {
req := &AssociateAssetsRequest{
SerialNumbers: []string{"SN1"},
ClientUserIds: []string{"user-1"},
}
err := req.Validate()
require.Error(t, err)
require.Contains(t, err.Error(), "mutually exclusive")
})
t.Run("neither populated is rejected", func(t *testing.T) {
req := &AssociateAssetsRequest{}
err := req.Validate()
require.Error(t, err)
require.Contains(t, err.Error(), "required")
})
}
func TestAssociateAssets(t *testing.T) {
tests := []struct {
name string
token string
params *AssociateAssetsRequest
handler http.HandlerFunc
expectedErrMsg string
}{
{
name: "valid request",
token: "valid_token",
params: &AssociateAssetsRequest{
Assets: []Asset{{AdamID: "12345", PricingParam: "STDQ"}},
SerialNumbers: []string{"SN12345"},
},
handler: func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, http.MethodPost, r.Method)
assert.Equal(t, "/assets/associate", r.URL.Path)
assert.Equal(t, "Bearer valid_token", r.Header.Get("Authorization"))
body, err := io.ReadAll(r.Body)
assert.NoError(t, err)
var reqParams AssociateAssetsRequest
err = json.Unmarshal(body, &reqParams)
assert.NoError(t, err)
assert.Equal(t, []Asset{{AdamID: "12345", PricingParam: "STDQ"}}, reqParams.Assets)
assert.Equal(t, []string{"SN12345"}, reqParams.SerialNumbers)
assert.Empty(t, reqParams.ClientUserIds)
// Verify omitempty: clientUserIds key should not appear in the wire payload.
assert.NotContains(t, string(body), "clientUserIds")
_, _ = w.Write([]byte(`{"eventId": "123"}`))
},
expectedErrMsg: "",
},
{
name: "valid request with client user ids",
token: "valid_token",
params: &AssociateAssetsRequest{
Assets: []Asset{{AdamID: "12345", PricingParam: "STDQ"}},
ClientUserIds: []string{"user-uuid-1", "user-uuid-2"},
},
handler: func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, http.MethodPost, r.Method)
assert.Equal(t, "/assets/associate", r.URL.Path)
assert.Equal(t, "Bearer valid_token", r.Header.Get("Authorization"))
body, err := io.ReadAll(r.Body)
assert.NoError(t, err)
var reqParams AssociateAssetsRequest
err = json.Unmarshal(body, &reqParams)
assert.NoError(t, err)
assert.Equal(t, []Asset{{AdamID: "12345", PricingParam: "STDQ"}}, reqParams.Assets)
assert.Empty(t, reqParams.SerialNumbers)
assert.Equal(t, []string{"user-uuid-1", "user-uuid-2"}, reqParams.ClientUserIds)
// Verify omitempty: serialNumbers key should not appear in the wire payload.
assert.NotContains(t, string(body), "serialNumbers")
_, _ = w.Write([]byte(`{"eventId": "456"}`))
},
expectedErrMsg: "",
},
{
name: "rejects both serials and client user ids before HTTP",
token: "valid_token",
params: &AssociateAssetsRequest{
Assets: []Asset{{AdamID: "12345", PricingParam: "STDQ"}},
SerialNumbers: []string{"SN12345"},
ClientUserIds: []string{"user-uuid-1"},
},
handler: func(w http.ResponseWriter, r *http.Request) {
t.Fatal("HTTP request must not be made when validation fails")
},
expectedErrMsg: "mutually exclusive",
},
{
name: "rejects neither serials nor client user ids before HTTP",
token: "valid_token",
params: &AssociateAssetsRequest{
Assets: []Asset{{AdamID: "12345", PricingParam: "STDQ"}},
},
handler: func(w http.ResponseWriter, r *http.Request) {
t.Fatal("HTTP request must not be made when validation fails")
},
expectedErrMsg: "required",
},
{
name: "server error",
token: "valid_token",
params: &AssociateAssetsRequest{
Assets: []Asset{{AdamID: "12345", PricingParam: "STDQ"}},
SerialNumbers: []string{"SN12345"},
},
handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintln(w, `Internal Server Error`)
},
expectedErrMsg: "calling Apple VPP endpoint failed with status 500: Internal Server Error\n",
},
{
name: "client error",
token: "valid_token",
params: &AssociateAssetsRequest{
Assets: []Asset{{AdamID: "12345", PricingParam: "STDQ"}},
SerialNumbers: []string{"SN12345"},
},
handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadRequest)
fmt.Fprintln(w, `{"errorInfo":{},"errorMessage":"Bad Request","errorNumber":400}`)
},
expectedErrMsg: "making request to Apple VPP endpoint: Apple VPP endpoint returned error: Bad Request (error number: 400)",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
setupFakeServer(t, tt.handler)
_, err := AssociateAssets(t.Context(), tt.token, tt.params)
if tt.expectedErrMsg != "" {
require.Error(t, err)
require.Contains(t, err.Error(), tt.expectedErrMsg)
} else {
require.NoError(t, err)
}
})
}
}
func TestGetAssets(t *testing.T) {
originalClient := client
client = fleethttp.NewClient(fleethttp.WithTimeout(time.Second))
t.Cleanup(func() {
client = originalClient
})
var requestCount atomic.Int64
tests := []struct {
name string
token string
filter *AssetFilter
handler http.HandlerFunc
expectedAssets []Asset
expectedErrMsg string
expectedRequests int
}{
{
name: "valid token and filters",
token: "valid_token",
filter: &AssetFilter{
AdamID: "12345",
},
handler: func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, http.MethodGet, r.Method)
require.Equal(t, "/assets", r.URL.Path)
require.Equal(t, "Bearer valid_token", r.Header.Get("Authorization"))
query := r.URL.Query()
require.Equal(t, "12345", query.Get("adamId"))
type resp struct {
Assets []Asset `json:"assets"`
}
assets := resp{
Assets: []Asset{
{AdamID: "12345", PricingParam: "STDQ"},
{AdamID: "67890", PricingParam: "PLUS"},
},
}
w.WriteHeader(http.StatusOK)
require.NoError(t, json.NewEncoder(w).Encode(assets))
},
expectedAssets: []Asset{
{AdamID: "12345", PricingParam: "STDQ"},
{AdamID: "67890", PricingParam: "PLUS"},
},
expectedErrMsg: "",
expectedRequests: 1,
},
{
name: "server error",
token: "valid_token",
filter: nil,
handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintln(w, `Internal Server Error`)
},
expectedAssets: nil,
expectedErrMsg: "calling Apple VPP endpoint failed with status 500: Internal Server Error\n",
expectedRequests: 1,
},
{
name: "client error",
token: "valid_token",
filter: nil,
handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadRequest)
fmt.Fprintln(w, `{"errorInfo":{},"errorMessage":"Bad Request","errorNumber":400}`)
},
expectedAssets: nil,
expectedErrMsg: "retrieving assets: Apple VPP endpoint returned error: Bad Request (error number: 400)",
expectedRequests: 1,
},
{
name: "always times out",
token: "valid_token",
filter: nil,
handler: func(w http.ResponseWriter, r *http.Request) {
time.Sleep(time.Second + 500*time.Millisecond) // longer than the 1s client timeout
type resp struct {
Assets []Asset `json:"assets"`
}
assets := resp{
Assets: []Asset{
{AdamID: "12345", PricingParam: "STDQ"},
{AdamID: "67890", PricingParam: "PLUS"},
},
}
w.WriteHeader(http.StatusOK)
require.NoError(t, json.NewEncoder(w).Encode(assets))
},
expectedAssets: nil,
expectedErrMsg: "exceeded",
expectedRequests: 3,
},
{
name: "times out then valid",
token: "valid_token",
filter: nil,
handler: func(w http.ResponseWriter, r *http.Request) {
if requestCount.Load() < 2 {
time.Sleep(time.Second + 500*time.Millisecond) // longer than the 1s client timeout
}
type resp struct {
Assets []Asset `json:"assets"`
}
assets := resp{
Assets: []Asset{
{AdamID: "12345", PricingParam: "STDQ"},
{AdamID: "67890", PricingParam: "PLUS"},
},
}
w.WriteHeader(http.StatusOK)
require.NoError(t, json.NewEncoder(w).Encode(assets))
},
expectedAssets: []Asset{
{AdamID: "12345", PricingParam: "STDQ"},
{AdamID: "67890", PricingParam: "PLUS"},
},
expectedErrMsg: "",
expectedRequests: 2,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
requestCount.Store(0)
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestCount.Add(1)
tt.handler(w, r)
})
setupFakeServer(t, h)
assets, err := GetAssets(t.Context(), tt.token, tt.filter)
if tt.expectedErrMsg != "" {
require.Error(t, err)
require.Contains(t, err.Error(), tt.expectedErrMsg)
} else {
require.NoError(t, err)
require.Equal(t, tt.expectedAssets, assets)
}
require.EqualValues(t, tt.expectedRequests, requestCount.Load())
})
}
}
func TestDoRetryAfter(t *testing.T) {
tests := []struct {
name string
handler http.HandlerFunc
wantCalls int
wantErr bool
}{
{
name: "no retry-after header",
handler: func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
_, err := w.Write([]byte("{}"))
require.NoError(t, err)
},
wantCalls: 1,
wantErr: true,
},
{
name: "invalid retry-after header",
handler: func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Retry-After", "foo")
w.WriteHeader(http.StatusInternalServerError)
_, err := w.Write([]byte("{}"))
require.NoError(t, err)
},
wantCalls: 1,
wantErr: true,
},
{
name: "three retries",
handler: func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Retry-After", "1")
w.WriteHeader(http.StatusInternalServerError)
_, err := w.Write([]byte("{}"))
require.NoError(t, err)
},
wantCalls: 3,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var calls int
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
calls++
if calls < tt.wantCalls {
tt.handler(w, r)
return
}
})
start := time.Now()
req, err := http.NewRequest(http.MethodGet, dev_mode.Env("FLEET_DEV_VPP_URL"), nil)
require.NoError(t, err)
err = do[any](req, "test-token", nil)
require.NoError(t, err)
require.Equal(t, tt.wantCalls, calls)
require.WithinRange(t, time.Now(), start, start.Add(time.Duration(tt.wantCalls)*time.Second))
})
}
}
func TestDoRetry(t *testing.T) {
t.Run("retries after 500 with Retry-After", func(t *testing.T) {
var calls int
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
calls++
// Verify Authorization header appears exactly once
authHeaders := r.Header.Values("Authorization")
require.Len(t, authHeaders, 1,
"expected exactly 1 Authorization header on attempt %d, got %d: %v",
calls, len(authHeaders), authHeaders)
require.Equal(t, "Bearer test-token", authHeaders[0])
// Verify POST body is intact
body, err := io.ReadAll(r.Body)
require.NoError(t, err)
require.NotEmpty(t, body, "request body should not be empty on attempt %d", calls)
var reqParams AssociateAssetsRequest
err = json.Unmarshal(body, &reqParams)
require.NoError(t, err, "request body should be valid JSON on attempt %d, got: %q", calls, string(body))
require.Equal(t, "462054704", reqParams.Assets[0].AdamID)
require.Equal(t, "GXH409KH7X", reqParams.SerialNumbers[0])
if calls == 1 {
// First call: return 500 with Retry-After to trigger retry
w.Header().Set("Retry-After", "1")
w.WriteHeader(http.StatusInternalServerError)
_, _ = w.Write([]byte("{}"))
return
}
// Second call: success
_, _ = w.Write([]byte(`{"eventId": "evt-123"}`))
})
eventID, err := AssociateAssets(t.Context(), "test-token", &AssociateAssetsRequest{
Assets: []Asset{{AdamID: "462054704", PricingParam: "STDQ"}},
SerialNumbers: []string{"GXH409KH7X"},
})
require.NoError(t, err)
require.Equal(t, "evt-123", eventID)
require.Equal(t, 2, calls)
})
t.Run("retries after error 9646", func(t *testing.T) {
var calls int
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
calls++
// Verify Authorization header appears exactly once
authHeaders := r.Header.Values("Authorization")
require.Len(t, authHeaders, 1,
"expected exactly 1 Authorization header on attempt %d, got %d: %v",
calls, len(authHeaders), authHeaders)
// Verify POST body is intact
body, err := io.ReadAll(r.Body)
require.NoError(t, err)
require.NotEmpty(t, body, "request body should not be empty on attempt %d", calls)
var reqParams AssociateAssetsRequest
err = json.Unmarshal(body, &reqParams)
require.NoError(t, err, "request body should be valid JSON on attempt %d, got: %q", calls, string(body))
require.Equal(t, "462054704", reqParams.Assets[0].AdamID)
if calls == 1 {
// First call: return rate-limit error 9646
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"errorMessage":"Too many requests","errorNumber":9646}`))
return
}
// Second call: success
_, _ = w.Write([]byte(`{"eventId": "evt-456"}`))
})
eventID, err := AssociateAssets(t.Context(), "test-token", &AssociateAssetsRequest{
Assets: []Asset{{AdamID: "462054704", PricingParam: "STDQ"}},
SerialNumbers: []string{"GXH409KH7X"},
})
require.NoError(t, err)
require.Equal(t, "evt-456", eventID)
require.GreaterOrEqual(t, calls, 2)
})
}
func TestRegisterUser(t *testing.T) {
t.Run("rejects empty client user id or managed apple id", func(t *testing.T) {
_, err := RegisterUser(t.Context(), "tok", "", "user@example.com")
require.Error(t, err)
_, err = RegisterUser(t.Context(), "tok", "uuid-1", "")
require.Error(t, err)
})
t.Run("success returns apple userId synchronously", func(t *testing.T) {
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, http.MethodPost, r.Method)
assert.Equal(t, "/registerVPPUserSrv", r.URL.Path)
// v1 carries the token in the body, not the Authorization header.
assert.Empty(t, r.Header.Get("Authorization"))
assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
body, err := io.ReadAll(r.Body)
assert.NoError(t, err)
var got struct {
SToken string `json:"sToken"`
ClientUserIDStr string `json:"clientUserIdStr"`
ManagedAppleIDStr string `json:"managedAppleIDStr"`
Email string `json:"email"`
}
assert.NoError(t, json.Unmarshal(body, &got))
assert.Equal(t, "valid_token", got.SToken)
assert.Equal(t, "uuid-1", got.ClientUserIDStr)
assert.Equal(t, "user1@example.com", got.ManagedAppleIDStr)
// Apple keys on email — Fleet sends the Managed Apple ID for both.
assert.Equal(t, "user1@example.com", got.Email)
_, _ = w.Write([]byte(`{
"status": 0,
"user": {
"userId": 12345,
"status": "Registered",
"clientUserIdStr": "uuid-1",
"managedAppleIDStr": "user1@example.com"
}
}`))
})
appleUserID, err := RegisterUser(t.Context(), "valid_token", "uuid-1", "user1@example.com")
require.NoError(t, err)
require.Equal(t, "12345", appleUserID)
})
t.Run("apple application error surfaces as ErrorResponse", func(t *testing.T) {
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
// v1 application errors come back as HTTP 200 with status=-1.
_, _ = w.Write([]byte(`{
"status": -1,
"errorNumber": 9637,
"errorMessage": "Managed Apple ID not found"
}`))
})
_, err := RegisterUser(t.Context(), "valid_token", "uuid-1", "missing@example.com")
require.Error(t, err)
var appleErr *ErrorResponse
require.ErrorAs(t, err, &appleErr)
require.EqualValues(t, 9637, appleErr.ErrorNumber)
require.Equal(t, "Managed Apple ID not found", appleErr.ErrorMessage)
})
t.Run("apple transport-level error surfaces as ErrorResponse", func(t *testing.T) {
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadRequest)
_, _ = w.Write([]byte(`{"errorMessage":"Bad Request","errorNumber":400}`))
})
_, err := RegisterUser(t.Context(), "valid_token", "uuid-1", "user1@example.com")
require.Error(t, err)
require.Contains(t, err.Error(), "error number: 400")
})
t.Run("success without user object is treated as error", func(t *testing.T) {
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"status": 0}`))
})
_, err := RegisterUser(t.Context(), "valid_token", "uuid-1", "user1@example.com")
require.Error(t, err)
require.Contains(t, err.Error(), "no user record")
})
}
// associateAssetsParams is a small valid request used by the retry tests below.
func associateAssetsParams() *AssociateAssetsRequest {
return &AssociateAssetsRequest{
Assets: []Asset{{AdamID: "1", PricingParam: "STDQ"}},
SerialNumbers: []string{"SN1"},
}
}
// TestDoRetryIsBoundedAndNonRecursive verifies that when Apple persistently
// returns a retryable condition, do() retries a BOUNDED number of times and
// returns — it must never recurse (which previously stacked open response
// bodies / cancel-watcher goroutines / spans / timers per level and OOM'd the
// server). It also verifies a sustained Retry-After stays bounded and that
// context cancellation aborts the backoff promptly.
// See https://github.com/fleetdm/fleet/issues/46656.
func TestDoRetryIsBoundedAndNonRecursive(t *testing.T) {
// Shrink the retry knobs so the bounded loop runs fast.
origAttempts, origBackoff, origInterval, origMult := vppMaxAttempts, maxVPPBackoff, vppRateLimitInterval, vppRateLimitBackoffMultiplier
t.Cleanup(func() {
vppMaxAttempts, maxVPPBackoff, vppRateLimitInterval, vppRateLimitBackoffMultiplier = origAttempts, origBackoff, origInterval, origMult
})
vppMaxAttempts = 4
vppRateLimitInterval = 1 * time.Millisecond
maxVPPBackoff = 5 * time.Millisecond
vppRateLimitBackoffMultiplier = 2
t.Run("rate-limited (too many requests) retries a bounded number of times then fails", func(t *testing.T) {
var calls atomic.Int32
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"errorMessage":"Too many requests","errorNumber":9646}`))
})
ctx := t.Context()
done := make(chan error, 1)
go func() {
_, err := AssociateAssets(ctx, "tok", associateAssetsParams())
done <- err
}()
select {
case err := <-done:
require.Error(t, err)
require.Contains(t, err.Error(), "rate limited")
case <-time.After(5 * time.Second):
t.Fatal("AssociateAssets did not return — the retry loop is not bounded")
}
require.EqualValues(t, vppMaxAttempts, calls.Load(),
"expected exactly vppMaxAttempts requests; more means the retries are nesting/recursing")
})
t.Run("HTTP 500 + Retry-After is honored but capped and bounded", func(t *testing.T) {
var calls atomic.Int32
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
w.Header().Set("Retry-After", "600") // Apple asks for 10 minutes
w.WriteHeader(http.StatusInternalServerError)
})
start := time.Now()
_, err := AssociateAssets(t.Context(), "tok", associateAssetsParams())
require.Error(t, err)
// Bounded to vppMaxAttempts — a sustained Retry-After must NOT loop forever.
require.EqualValues(t, vppMaxAttempts, calls.Load())
// Retry-After is honored but capped at maxVPPBackoff (5ms here), so the
// call finishes far under the 600s Apple requested — a multi-minute value
// can't pin a synchronous request open.
require.Less(t, time.Since(start), 2*time.Second)
})
t.Run("context cancellation aborts the backoff promptly", func(t *testing.T) {
// Use a long backoff so that, without ctx cancellation, the call would block.
vppRateLimitInterval = 30 * time.Second
maxVPPBackoff = 30 * time.Second
t.Cleanup(func() {
vppRateLimitInterval = 1 * time.Millisecond
maxVPPBackoff = 5 * time.Millisecond
})
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"errorMessage":"Too many requests","errorNumber":9646}`))
})
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
defer cancel()
start := time.Now()
_, err := AssociateAssets(ctx, "tok", associateAssetsParams())
require.Error(t, err)
require.ErrorContains(t, err, "context")
require.Less(t, time.Since(start), 2*time.Second, "ctx cancellation should abort the backoff sleep")
})
t.Run("applies a growing backoff between retries", func(t *testing.T) {
// Override for measurable, non-flaky spacing; restore afterward.
oa, oi, om, ob := vppMaxAttempts, vppRateLimitInterval, vppRateLimitBackoffMultiplier, maxVPPBackoff
t.Cleanup(func() {
vppMaxAttempts, vppRateLimitInterval, vppRateLimitBackoffMultiplier, maxVPPBackoff = oa, oi, om, ob
})
vppMaxAttempts = 3
vppRateLimitInterval = 30 * time.Millisecond
vppRateLimitBackoffMultiplier = 2
maxVPPBackoff = time.Second // generous, so capping doesn't interfere here
var mu sync.Mutex
var times []time.Time
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
times = append(times, time.Now())
mu.Unlock()
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"errorMessage":"Too many requests","errorNumber":9646}`))
})
_, err := AssociateAssets(t.Context(), "tok", associateAssetsParams())
require.Error(t, err)
mu.Lock()
defer mu.Unlock()
require.Len(t, times, 3)
// The backoff is actually applied between attempts (not skipped) and
// grows (30ms, then 60ms). Timers fire at-or-after their interval, so
// these lower bounds are not flaky.
require.GreaterOrEqual(t, times[1].Sub(times[0]), 30*time.Millisecond)
require.GreaterOrEqual(t, times[2].Sub(times[1]), 60*time.Millisecond)
})
}
// TestDoVPPAttemptClampsRetryAfter verifies that an absurdly large Retry-After
// value is clamped to the backoff cap before being scaled to a time.Duration,
// rather than overflowing the int64 nanosecond math (which could wrap negative
// and bypass the cap). Uses the default maxVPPBackoff and calls doVPPAttempt
// directly (it doesn't sleep), so the test is instant.
func TestDoVPPAttemptClampsRetryAfter(t *testing.T) {
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
// ~1e14 seconds: seconds * 1e9 ns overflows int64 if not clamped first.
w.Header().Set("Retry-After", "99999999999999")
w.WriteHeader(http.StatusInternalServerError)
})
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, dev_mode.Env("FLEET_DEV_VPP_URL"), nil)
require.NoError(t, err)
done, retryAfter, err := doVPPAttempt[any](req, nil)
require.NoError(t, err)
require.False(t, done, "a 500 + Retry-After should be retryable, not terminal")
require.Greater(t, retryAfter, time.Duration(0), "clamped Retry-After must stay positive (no overflow to negative)")
require.Equal(t, maxVPPBackoff, retryAfter, "an over-cap Retry-After should clamp to the backoff cap")
}
func TestIsMaxDevicesPerUserError(t *testing.T) {
cases := []struct {
name string
err error
expected bool
}{
{
name: "nil error",
err: nil,
expected: false,
},
{
name: "non-VPP error",
err: errors.New("network down"),
expected: false,
},
{
name: "canonical numeric code 9622",
err: &ErrorResponse{ErrorMessage: "License count exceeded", ErrorNumber: 9622},
expected: true,
},
{
name: "matched by case-insensitive message",
err: &ErrorResponse{ErrorMessage: "User has reached the Maximum Number of Devices for this license", ErrorNumber: 99999},
expected: true,
},
{
name: "matched by 'device limit' phrasing",
err: &ErrorResponse{ErrorMessage: "Device limit exceeded for this client user.", ErrorNumber: 0},
expected: true,
},
{
name: "unrelated VPP error 9610",
err: &ErrorResponse{ErrorMessage: "Cannot establish a connection.", ErrorNumber: 9610},
expected: false,
},
{
name: "wrapped via fmt.Errorf %w still detected",
err: fmt.Errorf("calling vpp: %w", &ErrorResponse{ErrorMessage: "User has reached the maximum number of devices.", ErrorNumber: 9622}),
expected: true,
},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
require.Equal(t, tt.expected, IsMaxDevicesPerUserError(tt.err))
})
}
}
func TestGetBaseURL(t *testing.T) {
t.Run("Default URL", func(t *testing.T) {
require.Equal(t, "https://vpp.itunes.apple.com/mdm/v2", getBaseURL())
})
t.Run("Custom URL", func(t *testing.T) {
customURL := "http://localhost:8000"
dev_mode.SetOverride("FLEET_DEV_VPP_URL", customURL, t)
require.Equal(t, customURL, getBaseURL())
})
}