<!-- Add the related story/sub-task/bug number, like Resolves #123, or remove if NA --> **Related issue:** Resolves #46656 `server/mdm/apple/vpp.do` retried transient Apple errors by **calling itself recursively**, with the rate-limit branch nesting `retry.Do` inside `retry.Do`. This change replaces the recursion with a single retry loop (respecting the prior 1 initial attempt + 3 retries), closes each response before retrying, honors Apple's `Retry-After` capped at 30s so that a multi-minute value can't block a synchronous request, and threads `context` through the VPP calls so the backoff is cancellable. The retry timings are otherwise unchanged from before. Following @sgress454 suggestion, I considered routing this through the shared `retry.Do` helper (a single attempt wrapped in `retry.Do` + an error filter) but figured out that: - retry.Do` owns its own wait schedule and its error filter returns an outcome enum rather than a duration, so it can't honor Apple's per-response `Retry-After` value. - also, I'd have to change the `retry` package to receive an extra `ctx` param so that the backoff is context-aware (which IMHO is more blast radius than this incident fix should carry). # Checklist for submitter - [x] Changes file added for user-visible changes in `changes/`. - [x] Timeouts are implemented and retries are limited to avoid infinite loops ## Testing - [x] Added/updated automated tests - [x] QA'd all new/changed functionality manually **What was verified.** The new automated test cannot run against `main` (the fix changes the VPP function signatures and adds the retry knobs), so to confirm the actual failure mode I checked out `main` and ran a small repro that drives the VPP client against an Apple endpoint that always returns the rate-limit error. On `main`, the call **never returns** — `do()` recurses without bound — and the repro times out: ``` --- FAIL: TestReproUnboundedRecursionOnMain (10.00s) zz_repro_main_test.go:30: AssociateAssets did NOT return within 10s — unbounded retry recursion in do() on main FAIL FAIL github.com/fleetdm/fleet/v4/server/mdm/apple/vpp 10.642s ``` On this branch the same scenario returns a bounded error promptly. That behavior is covered by the new `TestDoRetryIsBoundedAndNonRecursive` (bounded rate-limit retries, `Retry-After` honored-but-capped, and context cancellation), and the full `server/mdm/apple/vpp` package passes. **I did not perform an end-to-end QA against a live Apple endpoint**. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **Bug Fixes** * Fixed a server out-of-memory crash that occurred when Apple VPP API repeatedly returned transient errors during VPP operations, including app installs, user registration, and license seat releases. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
902 lines
29 KiB
Go
902 lines
29 KiB
Go
package vpp
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/fleetdm/fleet/v4/pkg/fleethttp"
|
|
"github.com/fleetdm/fleet/v4/server/dev_mode"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func setupFakeServer(t *testing.T, handler http.HandlerFunc) {
|
|
server := httptest.NewServer(handler)
|
|
dev_mode.SetOverride("FLEET_DEV_VPP_URL", server.URL, t)
|
|
t.Cleanup(server.Close)
|
|
}
|
|
|
|
func TestGetConfig(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
token string
|
|
handler http.HandlerFunc
|
|
wantName string
|
|
wantCountry string
|
|
expectedErrMsg string
|
|
expectMinCalls int
|
|
expectMaxCalls int
|
|
}{
|
|
{
|
|
name: "valid token US",
|
|
token: "valid_token",
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
fmt.Fprintln(w, `{"locationName": "Test Location", "countryISO2ACode": "US"}`)
|
|
},
|
|
wantName: "Test Location",
|
|
wantCountry: "us",
|
|
expectedErrMsg: "",
|
|
expectMinCalls: 1,
|
|
expectMaxCalls: 1,
|
|
},
|
|
{
|
|
name: "valid token DE lowercased",
|
|
token: "valid_token",
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
fmt.Fprintln(w, `{"locationName": "DE Org", "countryISO2ACode": "DE"}`)
|
|
},
|
|
wantName: "DE Org",
|
|
wantCountry: "de",
|
|
expectedErrMsg: "",
|
|
expectMinCalls: 1,
|
|
expectMaxCalls: 1,
|
|
},
|
|
{
|
|
name: "invalid token",
|
|
token: "invalid_token",
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusUnauthorized)
|
|
fmt.Fprintln(w, `{"errorNumber": 9622}`)
|
|
},
|
|
wantName: "",
|
|
wantCountry: "",
|
|
expectedErrMsg: "making request to Apple VPP endpoint: Apple VPP endpoint returned error: (error number: 9622)",
|
|
// Apple application errors should not be retried.
|
|
expectMinCalls: 1,
|
|
expectMaxCalls: 1,
|
|
},
|
|
{
|
|
name: "server error retries up to 3 times",
|
|
token: "valid_token",
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
fmt.Fprintln(w, `Internal Server Error`)
|
|
},
|
|
wantName: "",
|
|
wantCountry: "",
|
|
expectedErrMsg: "calling Apple VPP endpoint failed with status 500: Internal Server Error\n",
|
|
expectMinCalls: 3,
|
|
expectMaxCalls: 3,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
var calls int
|
|
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
|
|
calls++
|
|
tt.handler(w, r)
|
|
})
|
|
|
|
cfg, err := GetConfig(t.Context(), tt.token)
|
|
if tt.expectedErrMsg != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), tt.expectedErrMsg)
|
|
} else {
|
|
require.NoError(t, err)
|
|
}
|
|
require.Equal(t, tt.wantName, cfg.LocationName)
|
|
require.Equal(t, tt.wantCountry, cfg.CountryCode)
|
|
if tt.expectMinCalls > 0 {
|
|
require.GreaterOrEqual(t, calls, tt.expectMinCalls)
|
|
require.LessOrEqual(t, calls, tt.expectMaxCalls)
|
|
}
|
|
})
|
|
}
|
|
|
|
t.Run("transient failure then success", func(t *testing.T) {
|
|
var calls int
|
|
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
|
|
calls++
|
|
if calls < 2 {
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
fmt.Fprintln(w, `Internal Server Error`)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusOK)
|
|
fmt.Fprintln(w, `{"locationName": "Recovered", "countryISO2ACode": "FR"}`)
|
|
})
|
|
|
|
cfg, err := GetConfig(t.Context(), "token")
|
|
require.NoError(t, err)
|
|
require.Equal(t, "Recovered", cfg.LocationName)
|
|
require.Equal(t, "fr", cfg.CountryCode)
|
|
require.Equal(t, 2, calls)
|
|
})
|
|
}
|
|
|
|
func TestAssociateAssetsRequestValidate(t *testing.T) {
|
|
t.Run("serial numbers only is valid", func(t *testing.T) {
|
|
req := &AssociateAssetsRequest{SerialNumbers: []string{"SN1"}}
|
|
require.NoError(t, req.Validate())
|
|
})
|
|
t.Run("client user ids only is valid", func(t *testing.T) {
|
|
req := &AssociateAssetsRequest{ClientUserIds: []string{"user-1"}}
|
|
require.NoError(t, req.Validate())
|
|
})
|
|
t.Run("both populated is rejected", func(t *testing.T) {
|
|
req := &AssociateAssetsRequest{
|
|
SerialNumbers: []string{"SN1"},
|
|
ClientUserIds: []string{"user-1"},
|
|
}
|
|
err := req.Validate()
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), "mutually exclusive")
|
|
})
|
|
t.Run("neither populated is rejected", func(t *testing.T) {
|
|
req := &AssociateAssetsRequest{}
|
|
err := req.Validate()
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), "required")
|
|
})
|
|
}
|
|
|
|
func TestAssociateAssets(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
token string
|
|
params *AssociateAssetsRequest
|
|
handler http.HandlerFunc
|
|
expectedErrMsg string
|
|
}{
|
|
{
|
|
name: "valid request",
|
|
token: "valid_token",
|
|
params: &AssociateAssetsRequest{
|
|
Assets: []Asset{{AdamID: "12345", PricingParam: "STDQ"}},
|
|
SerialNumbers: []string{"SN12345"},
|
|
},
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
assert.Equal(t, http.MethodPost, r.Method)
|
|
assert.Equal(t, "/assets/associate", r.URL.Path)
|
|
assert.Equal(t, "Bearer valid_token", r.Header.Get("Authorization"))
|
|
|
|
body, err := io.ReadAll(r.Body)
|
|
assert.NoError(t, err)
|
|
|
|
var reqParams AssociateAssetsRequest
|
|
err = json.Unmarshal(body, &reqParams)
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, []Asset{{AdamID: "12345", PricingParam: "STDQ"}}, reqParams.Assets)
|
|
assert.Equal(t, []string{"SN12345"}, reqParams.SerialNumbers)
|
|
assert.Empty(t, reqParams.ClientUserIds)
|
|
|
|
// Verify omitempty: clientUserIds key should not appear in the wire payload.
|
|
assert.NotContains(t, string(body), "clientUserIds")
|
|
|
|
_, _ = w.Write([]byte(`{"eventId": "123"}`))
|
|
},
|
|
expectedErrMsg: "",
|
|
},
|
|
{
|
|
name: "valid request with client user ids",
|
|
token: "valid_token",
|
|
params: &AssociateAssetsRequest{
|
|
Assets: []Asset{{AdamID: "12345", PricingParam: "STDQ"}},
|
|
ClientUserIds: []string{"user-uuid-1", "user-uuid-2"},
|
|
},
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
assert.Equal(t, http.MethodPost, r.Method)
|
|
assert.Equal(t, "/assets/associate", r.URL.Path)
|
|
assert.Equal(t, "Bearer valid_token", r.Header.Get("Authorization"))
|
|
|
|
body, err := io.ReadAll(r.Body)
|
|
assert.NoError(t, err)
|
|
|
|
var reqParams AssociateAssetsRequest
|
|
err = json.Unmarshal(body, &reqParams)
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, []Asset{{AdamID: "12345", PricingParam: "STDQ"}}, reqParams.Assets)
|
|
assert.Empty(t, reqParams.SerialNumbers)
|
|
assert.Equal(t, []string{"user-uuid-1", "user-uuid-2"}, reqParams.ClientUserIds)
|
|
|
|
// Verify omitempty: serialNumbers key should not appear in the wire payload.
|
|
assert.NotContains(t, string(body), "serialNumbers")
|
|
|
|
_, _ = w.Write([]byte(`{"eventId": "456"}`))
|
|
},
|
|
expectedErrMsg: "",
|
|
},
|
|
{
|
|
name: "rejects both serials and client user ids before HTTP",
|
|
token: "valid_token",
|
|
params: &AssociateAssetsRequest{
|
|
Assets: []Asset{{AdamID: "12345", PricingParam: "STDQ"}},
|
|
SerialNumbers: []string{"SN12345"},
|
|
ClientUserIds: []string{"user-uuid-1"},
|
|
},
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
t.Fatal("HTTP request must not be made when validation fails")
|
|
},
|
|
expectedErrMsg: "mutually exclusive",
|
|
},
|
|
{
|
|
name: "rejects neither serials nor client user ids before HTTP",
|
|
token: "valid_token",
|
|
params: &AssociateAssetsRequest{
|
|
Assets: []Asset{{AdamID: "12345", PricingParam: "STDQ"}},
|
|
},
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
t.Fatal("HTTP request must not be made when validation fails")
|
|
},
|
|
expectedErrMsg: "required",
|
|
},
|
|
{
|
|
name: "server error",
|
|
token: "valid_token",
|
|
params: &AssociateAssetsRequest{
|
|
Assets: []Asset{{AdamID: "12345", PricingParam: "STDQ"}},
|
|
SerialNumbers: []string{"SN12345"},
|
|
},
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
fmt.Fprintln(w, `Internal Server Error`)
|
|
},
|
|
expectedErrMsg: "calling Apple VPP endpoint failed with status 500: Internal Server Error\n",
|
|
},
|
|
{
|
|
name: "client error",
|
|
token: "valid_token",
|
|
params: &AssociateAssetsRequest{
|
|
Assets: []Asset{{AdamID: "12345", PricingParam: "STDQ"}},
|
|
SerialNumbers: []string{"SN12345"},
|
|
},
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
fmt.Fprintln(w, `{"errorInfo":{},"errorMessage":"Bad Request","errorNumber":400}`)
|
|
},
|
|
expectedErrMsg: "making request to Apple VPP endpoint: Apple VPP endpoint returned error: Bad Request (error number: 400)",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
setupFakeServer(t, tt.handler)
|
|
|
|
_, err := AssociateAssets(t.Context(), tt.token, tt.params)
|
|
if tt.expectedErrMsg != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), tt.expectedErrMsg)
|
|
} else {
|
|
require.NoError(t, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGetAssets(t *testing.T) {
|
|
originalClient := client
|
|
client = fleethttp.NewClient(fleethttp.WithTimeout(time.Second))
|
|
t.Cleanup(func() {
|
|
client = originalClient
|
|
})
|
|
|
|
var requestCount atomic.Int64
|
|
|
|
tests := []struct {
|
|
name string
|
|
token string
|
|
filter *AssetFilter
|
|
handler http.HandlerFunc
|
|
expectedAssets []Asset
|
|
expectedErrMsg string
|
|
expectedRequests int
|
|
}{
|
|
{
|
|
name: "valid token and filters",
|
|
token: "valid_token",
|
|
filter: &AssetFilter{
|
|
AdamID: "12345",
|
|
},
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
require.Equal(t, http.MethodGet, r.Method)
|
|
require.Equal(t, "/assets", r.URL.Path)
|
|
require.Equal(t, "Bearer valid_token", r.Header.Get("Authorization"))
|
|
|
|
query := r.URL.Query()
|
|
require.Equal(t, "12345", query.Get("adamId"))
|
|
|
|
type resp struct {
|
|
Assets []Asset `json:"assets"`
|
|
}
|
|
assets := resp{
|
|
Assets: []Asset{
|
|
{AdamID: "12345", PricingParam: "STDQ"},
|
|
{AdamID: "67890", PricingParam: "PLUS"},
|
|
},
|
|
}
|
|
w.WriteHeader(http.StatusOK)
|
|
require.NoError(t, json.NewEncoder(w).Encode(assets))
|
|
},
|
|
expectedAssets: []Asset{
|
|
{AdamID: "12345", PricingParam: "STDQ"},
|
|
{AdamID: "67890", PricingParam: "PLUS"},
|
|
},
|
|
expectedErrMsg: "",
|
|
expectedRequests: 1,
|
|
},
|
|
{
|
|
name: "server error",
|
|
token: "valid_token",
|
|
filter: nil,
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
fmt.Fprintln(w, `Internal Server Error`)
|
|
},
|
|
expectedAssets: nil,
|
|
expectedErrMsg: "calling Apple VPP endpoint failed with status 500: Internal Server Error\n",
|
|
expectedRequests: 1,
|
|
},
|
|
{
|
|
name: "client error",
|
|
token: "valid_token",
|
|
filter: nil,
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
fmt.Fprintln(w, `{"errorInfo":{},"errorMessage":"Bad Request","errorNumber":400}`)
|
|
},
|
|
expectedAssets: nil,
|
|
expectedErrMsg: "retrieving assets: Apple VPP endpoint returned error: Bad Request (error number: 400)",
|
|
expectedRequests: 1,
|
|
},
|
|
{
|
|
name: "always times out",
|
|
token: "valid_token",
|
|
filter: nil,
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
time.Sleep(time.Second + 500*time.Millisecond) // longer than the 1s client timeout
|
|
type resp struct {
|
|
Assets []Asset `json:"assets"`
|
|
}
|
|
assets := resp{
|
|
Assets: []Asset{
|
|
{AdamID: "12345", PricingParam: "STDQ"},
|
|
{AdamID: "67890", PricingParam: "PLUS"},
|
|
},
|
|
}
|
|
w.WriteHeader(http.StatusOK)
|
|
require.NoError(t, json.NewEncoder(w).Encode(assets))
|
|
},
|
|
expectedAssets: nil,
|
|
expectedErrMsg: "exceeded",
|
|
expectedRequests: 3,
|
|
},
|
|
{
|
|
name: "times out then valid",
|
|
token: "valid_token",
|
|
filter: nil,
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
if requestCount.Load() < 2 {
|
|
time.Sleep(time.Second + 500*time.Millisecond) // longer than the 1s client timeout
|
|
}
|
|
|
|
type resp struct {
|
|
Assets []Asset `json:"assets"`
|
|
}
|
|
assets := resp{
|
|
Assets: []Asset{
|
|
{AdamID: "12345", PricingParam: "STDQ"},
|
|
{AdamID: "67890", PricingParam: "PLUS"},
|
|
},
|
|
}
|
|
w.WriteHeader(http.StatusOK)
|
|
require.NoError(t, json.NewEncoder(w).Encode(assets))
|
|
},
|
|
expectedAssets: []Asset{
|
|
{AdamID: "12345", PricingParam: "STDQ"},
|
|
{AdamID: "67890", PricingParam: "PLUS"},
|
|
},
|
|
expectedErrMsg: "",
|
|
expectedRequests: 2,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
requestCount.Store(0)
|
|
|
|
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
requestCount.Add(1)
|
|
tt.handler(w, r)
|
|
})
|
|
setupFakeServer(t, h)
|
|
|
|
assets, err := GetAssets(t.Context(), tt.token, tt.filter)
|
|
if tt.expectedErrMsg != "" {
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), tt.expectedErrMsg)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, tt.expectedAssets, assets)
|
|
}
|
|
require.EqualValues(t, tt.expectedRequests, requestCount.Load())
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDoRetryAfter(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
handler http.HandlerFunc
|
|
wantCalls int
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "no retry-after header",
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
_, err := w.Write([]byte("{}"))
|
|
require.NoError(t, err)
|
|
},
|
|
wantCalls: 1,
|
|
wantErr: true,
|
|
},
|
|
{
|
|
name: "invalid retry-after header",
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Add("Retry-After", "foo")
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
_, err := w.Write([]byte("{}"))
|
|
require.NoError(t, err)
|
|
},
|
|
wantCalls: 1,
|
|
wantErr: true,
|
|
},
|
|
{
|
|
name: "three retries",
|
|
handler: func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Add("Retry-After", "1")
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
_, err := w.Write([]byte("{}"))
|
|
require.NoError(t, err)
|
|
},
|
|
wantCalls: 3,
|
|
wantErr: false,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
var calls int
|
|
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
|
|
calls++
|
|
if calls < tt.wantCalls {
|
|
tt.handler(w, r)
|
|
return
|
|
}
|
|
})
|
|
|
|
start := time.Now()
|
|
req, err := http.NewRequest(http.MethodGet, dev_mode.Env("FLEET_DEV_VPP_URL"), nil)
|
|
require.NoError(t, err)
|
|
err = do[any](req, "test-token", nil)
|
|
require.NoError(t, err)
|
|
require.Equal(t, tt.wantCalls, calls)
|
|
require.WithinRange(t, time.Now(), start, start.Add(time.Duration(tt.wantCalls)*time.Second))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDoRetry(t *testing.T) {
|
|
t.Run("retries after 500 with Retry-After", func(t *testing.T) {
|
|
var calls int
|
|
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
|
|
calls++
|
|
|
|
// Verify Authorization header appears exactly once
|
|
authHeaders := r.Header.Values("Authorization")
|
|
require.Len(t, authHeaders, 1,
|
|
"expected exactly 1 Authorization header on attempt %d, got %d: %v",
|
|
calls, len(authHeaders), authHeaders)
|
|
require.Equal(t, "Bearer test-token", authHeaders[0])
|
|
|
|
// Verify POST body is intact
|
|
body, err := io.ReadAll(r.Body)
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, body, "request body should not be empty on attempt %d", calls)
|
|
|
|
var reqParams AssociateAssetsRequest
|
|
err = json.Unmarshal(body, &reqParams)
|
|
require.NoError(t, err, "request body should be valid JSON on attempt %d, got: %q", calls, string(body))
|
|
require.Equal(t, "462054704", reqParams.Assets[0].AdamID)
|
|
require.Equal(t, "GXH409KH7X", reqParams.SerialNumbers[0])
|
|
|
|
if calls == 1 {
|
|
// First call: return 500 with Retry-After to trigger retry
|
|
w.Header().Set("Retry-After", "1")
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
_, _ = w.Write([]byte("{}"))
|
|
return
|
|
}
|
|
|
|
// Second call: success
|
|
_, _ = w.Write([]byte(`{"eventId": "evt-123"}`))
|
|
})
|
|
|
|
eventID, err := AssociateAssets(t.Context(), "test-token", &AssociateAssetsRequest{
|
|
Assets: []Asset{{AdamID: "462054704", PricingParam: "STDQ"}},
|
|
SerialNumbers: []string{"GXH409KH7X"},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, "evt-123", eventID)
|
|
require.Equal(t, 2, calls)
|
|
})
|
|
|
|
t.Run("retries after error 9646", func(t *testing.T) {
|
|
var calls int
|
|
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
|
|
calls++
|
|
|
|
// Verify Authorization header appears exactly once
|
|
authHeaders := r.Header.Values("Authorization")
|
|
require.Len(t, authHeaders, 1,
|
|
"expected exactly 1 Authorization header on attempt %d, got %d: %v",
|
|
calls, len(authHeaders), authHeaders)
|
|
|
|
// Verify POST body is intact
|
|
body, err := io.ReadAll(r.Body)
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, body, "request body should not be empty on attempt %d", calls)
|
|
|
|
var reqParams AssociateAssetsRequest
|
|
err = json.Unmarshal(body, &reqParams)
|
|
require.NoError(t, err, "request body should be valid JSON on attempt %d, got: %q", calls, string(body))
|
|
require.Equal(t, "462054704", reqParams.Assets[0].AdamID)
|
|
|
|
if calls == 1 {
|
|
// First call: return rate-limit error 9646
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(`{"errorMessage":"Too many requests","errorNumber":9646}`))
|
|
return
|
|
}
|
|
|
|
// Second call: success
|
|
_, _ = w.Write([]byte(`{"eventId": "evt-456"}`))
|
|
})
|
|
|
|
eventID, err := AssociateAssets(t.Context(), "test-token", &AssociateAssetsRequest{
|
|
Assets: []Asset{{AdamID: "462054704", PricingParam: "STDQ"}},
|
|
SerialNumbers: []string{"GXH409KH7X"},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, "evt-456", eventID)
|
|
require.GreaterOrEqual(t, calls, 2)
|
|
})
|
|
}
|
|
|
|
func TestRegisterUser(t *testing.T) {
|
|
t.Run("rejects empty client user id or managed apple id", func(t *testing.T) {
|
|
_, err := RegisterUser(t.Context(), "tok", "", "user@example.com")
|
|
require.Error(t, err)
|
|
|
|
_, err = RegisterUser(t.Context(), "tok", "uuid-1", "")
|
|
require.Error(t, err)
|
|
})
|
|
|
|
t.Run("success returns apple userId synchronously", func(t *testing.T) {
|
|
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
|
|
assert.Equal(t, http.MethodPost, r.Method)
|
|
assert.Equal(t, "/registerVPPUserSrv", r.URL.Path)
|
|
// v1 carries the token in the body, not the Authorization header.
|
|
assert.Empty(t, r.Header.Get("Authorization"))
|
|
assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
|
|
|
|
body, err := io.ReadAll(r.Body)
|
|
assert.NoError(t, err)
|
|
|
|
var got struct {
|
|
SToken string `json:"sToken"`
|
|
ClientUserIDStr string `json:"clientUserIdStr"`
|
|
ManagedAppleIDStr string `json:"managedAppleIDStr"`
|
|
Email string `json:"email"`
|
|
}
|
|
assert.NoError(t, json.Unmarshal(body, &got))
|
|
assert.Equal(t, "valid_token", got.SToken)
|
|
assert.Equal(t, "uuid-1", got.ClientUserIDStr)
|
|
assert.Equal(t, "user1@example.com", got.ManagedAppleIDStr)
|
|
// Apple keys on email — Fleet sends the Managed Apple ID for both.
|
|
assert.Equal(t, "user1@example.com", got.Email)
|
|
|
|
_, _ = w.Write([]byte(`{
|
|
"status": 0,
|
|
"user": {
|
|
"userId": 12345,
|
|
"status": "Registered",
|
|
"clientUserIdStr": "uuid-1",
|
|
"managedAppleIDStr": "user1@example.com"
|
|
}
|
|
}`))
|
|
})
|
|
|
|
appleUserID, err := RegisterUser(t.Context(), "valid_token", "uuid-1", "user1@example.com")
|
|
require.NoError(t, err)
|
|
require.Equal(t, "12345", appleUserID)
|
|
})
|
|
|
|
t.Run("apple application error surfaces as ErrorResponse", func(t *testing.T) {
|
|
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
|
|
// v1 application errors come back as HTTP 200 with status=-1.
|
|
_, _ = w.Write([]byte(`{
|
|
"status": -1,
|
|
"errorNumber": 9637,
|
|
"errorMessage": "Managed Apple ID not found"
|
|
}`))
|
|
})
|
|
|
|
_, err := RegisterUser(t.Context(), "valid_token", "uuid-1", "missing@example.com")
|
|
require.Error(t, err)
|
|
|
|
var appleErr *ErrorResponse
|
|
require.ErrorAs(t, err, &appleErr)
|
|
require.EqualValues(t, 9637, appleErr.ErrorNumber)
|
|
require.Equal(t, "Managed Apple ID not found", appleErr.ErrorMessage)
|
|
})
|
|
|
|
t.Run("apple transport-level error surfaces as ErrorResponse", func(t *testing.T) {
|
|
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
_, _ = w.Write([]byte(`{"errorMessage":"Bad Request","errorNumber":400}`))
|
|
})
|
|
|
|
_, err := RegisterUser(t.Context(), "valid_token", "uuid-1", "user1@example.com")
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), "error number: 400")
|
|
})
|
|
|
|
t.Run("success without user object is treated as error", func(t *testing.T) {
|
|
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = w.Write([]byte(`{"status": 0}`))
|
|
})
|
|
|
|
_, err := RegisterUser(t.Context(), "valid_token", "uuid-1", "user1@example.com")
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), "no user record")
|
|
})
|
|
}
|
|
|
|
// associateAssetsParams is a small valid request used by the retry tests below.
|
|
func associateAssetsParams() *AssociateAssetsRequest {
|
|
return &AssociateAssetsRequest{
|
|
Assets: []Asset{{AdamID: "1", PricingParam: "STDQ"}},
|
|
SerialNumbers: []string{"SN1"},
|
|
}
|
|
}
|
|
|
|
// TestDoRetryIsBoundedAndNonRecursive verifies that when Apple persistently
|
|
// returns a retryable condition, do() retries a BOUNDED number of times and
|
|
// returns — it must never recurse (which previously stacked open response
|
|
// bodies / cancel-watcher goroutines / spans / timers per level and OOM'd the
|
|
// server). It also verifies a sustained Retry-After stays bounded and that
|
|
// context cancellation aborts the backoff promptly.
|
|
// See https://github.com/fleetdm/fleet/issues/46656.
|
|
func TestDoRetryIsBoundedAndNonRecursive(t *testing.T) {
|
|
// Shrink the retry knobs so the bounded loop runs fast.
|
|
origAttempts, origBackoff, origInterval, origMult := vppMaxAttempts, maxVPPBackoff, vppRateLimitInterval, vppRateLimitBackoffMultiplier
|
|
t.Cleanup(func() {
|
|
vppMaxAttempts, maxVPPBackoff, vppRateLimitInterval, vppRateLimitBackoffMultiplier = origAttempts, origBackoff, origInterval, origMult
|
|
})
|
|
vppMaxAttempts = 4
|
|
vppRateLimitInterval = 1 * time.Millisecond
|
|
maxVPPBackoff = 5 * time.Millisecond
|
|
vppRateLimitBackoffMultiplier = 2
|
|
|
|
t.Run("rate-limited (too many requests) retries a bounded number of times then fails", func(t *testing.T) {
|
|
var calls atomic.Int32
|
|
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
|
|
calls.Add(1)
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(`{"errorMessage":"Too many requests","errorNumber":9646}`))
|
|
})
|
|
|
|
ctx := t.Context()
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
_, err := AssociateAssets(ctx, "tok", associateAssetsParams())
|
|
done <- err
|
|
}()
|
|
|
|
select {
|
|
case err := <-done:
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), "rate limited")
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("AssociateAssets did not return — the retry loop is not bounded")
|
|
}
|
|
|
|
require.EqualValues(t, vppMaxAttempts, calls.Load(),
|
|
"expected exactly vppMaxAttempts requests; more means the retries are nesting/recursing")
|
|
})
|
|
|
|
t.Run("HTTP 500 + Retry-After is honored but capped and bounded", func(t *testing.T) {
|
|
var calls atomic.Int32
|
|
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
|
|
calls.Add(1)
|
|
w.Header().Set("Retry-After", "600") // Apple asks for 10 minutes
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
})
|
|
|
|
start := time.Now()
|
|
_, err := AssociateAssets(t.Context(), "tok", associateAssetsParams())
|
|
require.Error(t, err)
|
|
// Bounded to vppMaxAttempts — a sustained Retry-After must NOT loop forever.
|
|
require.EqualValues(t, vppMaxAttempts, calls.Load())
|
|
// Retry-After is honored but capped at maxVPPBackoff (5ms here), so the
|
|
// call finishes far under the 600s Apple requested — a multi-minute value
|
|
// can't pin a synchronous request open.
|
|
require.Less(t, time.Since(start), 2*time.Second)
|
|
})
|
|
|
|
t.Run("context cancellation aborts the backoff promptly", func(t *testing.T) {
|
|
// Use a long backoff so that, without ctx cancellation, the call would block.
|
|
vppRateLimitInterval = 30 * time.Second
|
|
maxVPPBackoff = 30 * time.Second
|
|
t.Cleanup(func() {
|
|
vppRateLimitInterval = 1 * time.Millisecond
|
|
maxVPPBackoff = 5 * time.Millisecond
|
|
})
|
|
|
|
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(`{"errorMessage":"Too many requests","errorNumber":9646}`))
|
|
})
|
|
|
|
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
|
defer cancel()
|
|
|
|
start := time.Now()
|
|
_, err := AssociateAssets(ctx, "tok", associateAssetsParams())
|
|
require.Error(t, err)
|
|
require.ErrorContains(t, err, "context")
|
|
require.Less(t, time.Since(start), 2*time.Second, "ctx cancellation should abort the backoff sleep")
|
|
})
|
|
|
|
t.Run("applies a growing backoff between retries", func(t *testing.T) {
|
|
// Override for measurable, non-flaky spacing; restore afterward.
|
|
oa, oi, om, ob := vppMaxAttempts, vppRateLimitInterval, vppRateLimitBackoffMultiplier, maxVPPBackoff
|
|
t.Cleanup(func() {
|
|
vppMaxAttempts, vppRateLimitInterval, vppRateLimitBackoffMultiplier, maxVPPBackoff = oa, oi, om, ob
|
|
})
|
|
vppMaxAttempts = 3
|
|
vppRateLimitInterval = 30 * time.Millisecond
|
|
vppRateLimitBackoffMultiplier = 2
|
|
maxVPPBackoff = time.Second // generous, so capping doesn't interfere here
|
|
|
|
var mu sync.Mutex
|
|
var times []time.Time
|
|
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
|
|
mu.Lock()
|
|
times = append(times, time.Now())
|
|
mu.Unlock()
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte(`{"errorMessage":"Too many requests","errorNumber":9646}`))
|
|
})
|
|
|
|
_, err := AssociateAssets(t.Context(), "tok", associateAssetsParams())
|
|
require.Error(t, err)
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
require.Len(t, times, 3)
|
|
// The backoff is actually applied between attempts (not skipped) and
|
|
// grows (30ms, then 60ms). Timers fire at-or-after their interval, so
|
|
// these lower bounds are not flaky.
|
|
require.GreaterOrEqual(t, times[1].Sub(times[0]), 30*time.Millisecond)
|
|
require.GreaterOrEqual(t, times[2].Sub(times[1]), 60*time.Millisecond)
|
|
})
|
|
}
|
|
|
|
// TestDoVPPAttemptClampsRetryAfter verifies that an absurdly large Retry-After
|
|
// value is clamped to the backoff cap before being scaled to a time.Duration,
|
|
// rather than overflowing the int64 nanosecond math (which could wrap negative
|
|
// and bypass the cap). Uses the default maxVPPBackoff and calls doVPPAttempt
|
|
// directly (it doesn't sleep), so the test is instant.
|
|
func TestDoVPPAttemptClampsRetryAfter(t *testing.T) {
|
|
setupFakeServer(t, func(w http.ResponseWriter, r *http.Request) {
|
|
// ~1e14 seconds: seconds * 1e9 ns overflows int64 if not clamped first.
|
|
w.Header().Set("Retry-After", "99999999999999")
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
})
|
|
|
|
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, dev_mode.Env("FLEET_DEV_VPP_URL"), nil)
|
|
require.NoError(t, err)
|
|
|
|
done, retryAfter, err := doVPPAttempt[any](req, nil)
|
|
require.NoError(t, err)
|
|
require.False(t, done, "a 500 + Retry-After should be retryable, not terminal")
|
|
require.Greater(t, retryAfter, time.Duration(0), "clamped Retry-After must stay positive (no overflow to negative)")
|
|
require.Equal(t, maxVPPBackoff, retryAfter, "an over-cap Retry-After should clamp to the backoff cap")
|
|
}
|
|
|
|
func TestIsMaxDevicesPerUserError(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
err error
|
|
expected bool
|
|
}{
|
|
{
|
|
name: "nil error",
|
|
err: nil,
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "non-VPP error",
|
|
err: errors.New("network down"),
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "canonical numeric code 9622",
|
|
err: &ErrorResponse{ErrorMessage: "License count exceeded", ErrorNumber: 9622},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "matched by case-insensitive message",
|
|
err: &ErrorResponse{ErrorMessage: "User has reached the Maximum Number of Devices for this license", ErrorNumber: 99999},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "matched by 'device limit' phrasing",
|
|
err: &ErrorResponse{ErrorMessage: "Device limit exceeded for this client user.", ErrorNumber: 0},
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "unrelated VPP error 9610",
|
|
err: &ErrorResponse{ErrorMessage: "Cannot establish a connection.", ErrorNumber: 9610},
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "wrapped via fmt.Errorf %w still detected",
|
|
err: fmt.Errorf("calling vpp: %w", &ErrorResponse{ErrorMessage: "User has reached the maximum number of devices.", ErrorNumber: 9622}),
|
|
expected: true,
|
|
},
|
|
}
|
|
for _, tt := range cases {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
require.Equal(t, tt.expected, IsMaxDevicesPerUserError(tt.err))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGetBaseURL(t *testing.T) {
|
|
t.Run("Default URL", func(t *testing.T) {
|
|
require.Equal(t, "https://vpp.itunes.apple.com/mdm/v2", getBaseURL())
|
|
})
|
|
|
|
t.Run("Custom URL", func(t *testing.T) {
|
|
customURL := "http://localhost:8000"
|
|
dev_mode.SetOverride("FLEET_DEV_VPP_URL", customURL, t)
|
|
require.Equal(t, customURL, getBaseURL())
|
|
})
|
|
}
|