Files
fleet/client/orbit_client_test.go
T
Sharon Katz 7a419d6c3a Add exponential backoff to orbit config polling loop (#46674)
Closes #45627

Part 2 of #45553 -- see there for the full behavioral contract and
Oracle.

## Changes

- Integrated the shared `orbit/pkg/backoff` package (shipped in #45624)
into orbit's `ExecuteConfigReceivers` loop
- On error (5xx, network failure): polling interval doubles each failure
(30s, 60s, 120s, ...) capped at 5 minutes
- On success: resets immediately to normal 30s polling
- The inner `retry.Do` in `GetConfig` (transient retry within a single
tick) is unchanged

## Manual testing

### Automated tests

```
go test ./orbit/pkg/backoff/ -race -count=1   # 17 tests, 0 failures
go test ./client/ -count=1 -short              # client tests pass
```

### Build verification

```
go build ./orbit/cmd/orbit/     # compiles clean
go build ./orbit/cmd/desktop/   # compiles clean
```

### Dev environment testing

Built orbit from this branch and swapped it into a local dev setup
(`/opt/orbit/bin/orbit/macos/stable/orbit`). Server-side logs confirmed
that after the restart with the new binary, `/api/fleet/orbit/config`
requests stopped arriving at the fixed 30s cadence (old behavior),
consistent with backoff engaging on error responses. The `device_token`
endpoint (not covered by this PR) continued at its normal interval,
confirming the backoff is scoped to the config polling loop only.

Full end-to-end verification of the log messages (`backing off`,
`next_retry`, `exiting backoff`) should be done by QA with `sudo tail -f
/var/log/orbit/orbit.stderr.log`.

### QA manual test plan (cc @xpkoala)

**Setup:** Local Fleet server + orbit built from this branch (see build
steps above). Orbit logs are at `/var/log/orbit/orbit.stderr.log`
(requires `sudo`).

**Test 1 -- Backoff on server failure:**
1. Start Fleet server, verify orbit connects (config requests every ~30s
in server log)
2. Stop the Fleet server (`kill` the process or `docker stop` the
container)
3. Watch orbit logs: `sudo tail -f /var/log/orbit/orbit.stderr.log`
4. **Expected:** Log lines with `"running config receivers, backing
off"` and `next_retry` values increasing: ~60s, ~120s, ~240s, then
capping at ~5m (values include up to 10% random jitter)

**Test 2 -- Recovery resets to normal:**
1. While orbit is in backoff (from Test 1), restart the Fleet server
2. Wait for the next backoff tick to fire
3. **Expected:** Log line `"config receivers succeeded, exiting
backoff"` with `backoff_duration` showing how long the backoff lasted,
then polling resumes at normal 30s

**Test 3 -- Normal operation unchanged:**
1. With both server and orbit running healthy, watch orbit logs for ~2
minutes
2. **Expected:** No backoff-related log lines. Config polling stays at
30s intervals.

---

# Checklist for submitter

- [x] Changes file added for user-visible changes in `orbit/changes/`.
- [x] Input data is properly validated, no SQL changes, no JS changes.
- [x] Timeouts are implemented and retries are limited to avoid infinite
loops (backoff caps at 5 min).
- [x] Added/updated automated tests (existing backoff package tests
cover the mechanism).
- [ ] QA'd all new/changed functionality manually.

## fleetd/orbit/Fleet Desktop

- [x] If the change applies to only one platform, confirmed that
`runtime.GOOS` is used as needed to isolate changes (backoff is
platform-agnostic).
- [ ] Verified that fleetd runs on macOS, Linux and Windows.
- [ ] Verified auto-update works from the released version of component
to the new version.

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Improvements**
* Config polling now implements exponential backoff on network/server
failures, gradually increasing retry intervals up to a 5-minute maximum
instead of fixed intervals.
* After a successful config poll, the retry schedule automatically
resets back to the normal update interval.
* **Tests**
* Added unit tests to verify backoff increases after repeated failures
and resets promptly after recovery.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
2026-06-26 10:09:42 -04:00

271 lines
7.2 KiB
Go

package client
import (
"context"
"encoding/json"
"errors"
"reflect"
"testing"
"time"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetConfig(t *testing.T) {
t.Run(
"config cache", func(t *testing.T) {
oc := OrbitClient{}
oc.configCache.config = &fleet.OrbitConfig{}
oc.configCache.lastUpdated = time.Now().Add(1 * time.Second)
config, err := oc.GetConfig()
require.NoError(t, err)
require.Equal(t, oc.configCache.config, config)
},
)
t.Run(
"config cache error", func(t *testing.T) {
oc := OrbitClient{}
oc.configCache.config = nil
oc.configCache.err = errors.New("test error")
oc.configCache.lastUpdated = time.Now().Add(1 * time.Second)
config, err := oc.GetConfig()
require.Error(t, err)
require.Equal(t, oc.configCache.config, config)
},
)
}
func clientWithConfig(cfg *fleet.OrbitConfig) *OrbitClient {
ctx, cancel := context.WithCancel(context.Background())
oc := &OrbitClient{
receiverUpdateContext: ctx,
receiverUpdateCancelFunc: cancel,
}
oc.configCache.config = cfg
oc.configCache.lastUpdated = time.Now().Add(1 * time.Hour)
return oc
}
func TestConfigReceiverCalls(t *testing.T) {
var called1, called2 bool
testmsg := json.RawMessage("testing")
rfunc1 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
if !reflect.DeepEqual(cfg.Flags, testmsg) {
return errors.New("not equal testmsg")
}
called1 = true
return nil
})
rfunc2 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
if !reflect.DeepEqual(cfg.Flags, testmsg) {
return errors.New("not equal testmsg")
}
called2 = true
return nil
})
client := clientWithConfig(&fleet.OrbitConfig{Flags: testmsg})
client.RegisterConfigReceiver(rfunc1)
client.RegisterConfigReceiver(rfunc2)
err := client.RunConfigReceivers()
require.NoError(t, err)
require.True(t, called1)
require.True(t, called2)
}
func TestConfigReceiverErrors(t *testing.T) {
var called1, called2 bool
rfunc1 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
called1 = true
return nil
})
rfunc2 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
called2 = true
return nil
})
err1 := errors.New("error1")
err2 := errors.New("error2")
efunc1 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
return err1
})
efunc2 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
return err2
})
// Make sure we don't get stuck or crash on receiver panic
pfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
panic("woah")
})
client := clientWithConfig(&fleet.OrbitConfig{})
client.RegisterConfigReceiver(efunc1)
client.RegisterConfigReceiver(rfunc1)
client.RegisterConfigReceiver(efunc2)
client.RegisterConfigReceiver(rfunc2)
client.RegisterConfigReceiver(pfunc)
err := client.RunConfigReceivers()
require.ErrorIs(t, err, err1)
require.ErrorIs(t, err, err2)
require.True(t, called1)
require.True(t, called2)
}
func TestExecuteConfigReceiversCancel(t *testing.T) {
client := clientWithConfig(&fleet.OrbitConfig{})
client.ReceiverUpdateInterval = 100 * time.Millisecond
var calls1, calls2 int
requiredCalls := 4
cfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
calls1++
if calls1 == requiredCalls {
client.receiverUpdateCancelFunc()
}
return nil
})
rfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
calls2++
return nil
})
client.RegisterConfigReceiver(cfunc)
client.RegisterConfigReceiver(rfunc)
err := client.ExecuteConfigReceivers()
require.Nil(t, err)
require.Equal(t, requiredCalls, calls1)
require.Equal(t, requiredCalls, calls2)
}
func TestExecuteConfigReceiversInterrupt(t *testing.T) {
client := clientWithConfig(&fleet.OrbitConfig{})
defer client.receiverUpdateCancelFunc()
client.ReceiverUpdateInterval = 100 * time.Millisecond
var called bool
rfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
called = true
return nil
})
client.RegisterConfigReceiver(rfunc)
finChan := make(chan error)
go func() {
finChan <- client.ExecuteConfigReceivers()
}()
go func() {
time.Sleep(500 * time.Millisecond)
client.receiverUpdateCancelFunc()
}()
select {
case err := <-finChan:
require.Nil(t, err)
require.True(t, called)
case <-time.NewTimer(2 * time.Second).C:
require.Fail(t, "receiver interrupt cancel didn't work")
}
}
func TestExecuteConfigReceiversBackoffOnError(t *testing.T) {
client := clientWithConfig(&fleet.OrbitConfig{})
client.ReceiverUpdateInterval = 1 * time.Second
var callTimes []time.Time
callCount := 0
// 3 failures then cancel: intervals should be ~1s (base tick), ~2s, ~4s.
targetCalls := 4
rfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
callTimes = append(callTimes, time.Now())
callCount++
if callCount >= targetCalls {
client.receiverUpdateCancelFunc()
return nil
}
return errors.New("server error")
})
client.RegisterConfigReceiver(rfunc)
done := make(chan error, 1)
go func() { done <- client.ExecuteConfigReceivers() }()
select {
case err := <-done:
require.NoError(t, err)
case <-time.After(30 * time.Second):
t.Fatal("test timed out waiting for ExecuteConfigReceivers")
}
require.Equal(t, targetCalls, callCount)
// Verify each successive interval is strictly longer than the previous.
// Call 0->1 is the base tick (~1s), 1->2 should be ~2s, 2->3 should be ~4s.
require.GreaterOrEqual(t, len(callTimes), 3, "need at least 3 calls to verify growth")
for i := 1; i < len(callTimes)-1; i++ {
prev := callTimes[i].Sub(callTimes[i-1])
curr := callTimes[i+1].Sub(callTimes[i])
assert.Greater(t, curr, prev,
"interval %d->%d (%v) should be greater than %d->%d (%v)",
i, i+1, curr, i-1, i, prev)
}
}
func TestExecuteConfigReceiversResetOnSuccess(t *testing.T) {
client := clientWithConfig(&fleet.OrbitConfig{})
client.ReceiverUpdateInterval = 1 * time.Second
callCount := 0
var intervalAfterRecovery time.Duration
var recoveryStart time.Time
rfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
callCount++
switch {
case callCount <= 2:
// First 2 calls fail -- build up backoff
return errors.New("server error")
case callCount == 3:
// Success -- should reset backoff
recoveryStart = time.Now()
return nil
case callCount == 4:
// Next call should be at base interval (~1s), not backed off
intervalAfterRecovery = time.Since(recoveryStart)
client.receiverUpdateCancelFunc()
return nil
}
return nil
})
client.RegisterConfigReceiver(rfunc)
done := make(chan error, 1)
go func() { done <- client.ExecuteConfigReceivers() }()
select {
case err := <-done:
require.NoError(t, err)
case <-time.After(30 * time.Second):
t.Fatal("test timed out waiting for ExecuteConfigReceivers")
}
require.Equal(t, 4, callCount)
// After recovery, interval should be close to base (1s), not backed off.
// Use 2s as the upper bound: base (1s) + jitter (up to 10%) + scheduling slack.
assert.Less(t, intervalAfterRecovery, 2*time.Second,
"after success, interval should reset near base, got %v", intervalAfterRecovery)
}