Closes #45627 Part 2 of #45553 -- see there for the full behavioral contract and Oracle. ## Changes - Integrated the shared `orbit/pkg/backoff` package (shipped in #45624) into orbit's `ExecuteConfigReceivers` loop - On error (5xx, network failure): polling interval doubles each failure (30s, 60s, 120s, ...) capped at 5 minutes - On success: resets immediately to normal 30s polling - The inner `retry.Do` in `GetConfig` (transient retry within a single tick) is unchanged ## Manual testing ### Automated tests ``` go test ./orbit/pkg/backoff/ -race -count=1 # 17 tests, 0 failures go test ./client/ -count=1 -short # client tests pass ``` ### Build verification ``` go build ./orbit/cmd/orbit/ # compiles clean go build ./orbit/cmd/desktop/ # compiles clean ``` ### Dev environment testing Built orbit from this branch and swapped it into a local dev setup (`/opt/orbit/bin/orbit/macos/stable/orbit`). Server-side logs confirmed that after the restart with the new binary, `/api/fleet/orbit/config` requests stopped arriving at the fixed 30s cadence (old behavior), consistent with backoff engaging on error responses. The `device_token` endpoint (not covered by this PR) continued at its normal interval, confirming the backoff is scoped to the config polling loop only. Full end-to-end verification of the log messages (`backing off`, `next_retry`, `exiting backoff`) should be done by QA with `sudo tail -f /var/log/orbit/orbit.stderr.log`. ### QA manual test plan (cc @xpkoala) **Setup:** Local Fleet server + orbit built from this branch (see build steps above). Orbit logs are at `/var/log/orbit/orbit.stderr.log` (requires `sudo`). **Test 1 -- Backoff on server failure:** 1. Start Fleet server, verify orbit connects (config requests every ~30s in server log) 2. Stop the Fleet server (`kill` the process or `docker stop` the container) 3. Watch orbit logs: `sudo tail -f /var/log/orbit/orbit.stderr.log` 4. **Expected:** Log lines with `"running config receivers, backing off"` and `next_retry` values increasing: ~60s, ~120s, ~240s, then capping at ~5m (values include up to 10% random jitter) **Test 2 -- Recovery resets to normal:** 1. While orbit is in backoff (from Test 1), restart the Fleet server 2. Wait for the next backoff tick to fire 3. **Expected:** Log line `"config receivers succeeded, exiting backoff"` with `backoff_duration` showing how long the backoff lasted, then polling resumes at normal 30s **Test 3 -- Normal operation unchanged:** 1. With both server and orbit running healthy, watch orbit logs for ~2 minutes 2. **Expected:** No backoff-related log lines. Config polling stays at 30s intervals. --- # Checklist for submitter - [x] Changes file added for user-visible changes in `orbit/changes/`. - [x] Input data is properly validated, no SQL changes, no JS changes. - [x] Timeouts are implemented and retries are limited to avoid infinite loops (backoff caps at 5 min). - [x] Added/updated automated tests (existing backoff package tests cover the mechanism). - [ ] QA'd all new/changed functionality manually. ## fleetd/orbit/Fleet Desktop - [x] If the change applies to only one platform, confirmed that `runtime.GOOS` is used as needed to isolate changes (backoff is platform-agnostic). - [ ] Verified that fleetd runs on macOS, Linux and Windows. - [ ] Verified auto-update works from the released version of component to the new version. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **Improvements** * Config polling now implements exponential backoff on network/server failures, gradually increasing retry intervals up to a 5-minute maximum instead of fixed intervals. * After a successful config poll, the retry schedule automatically resets back to the normal update interval. * **Tests** * Added unit tests to verify backoff increases after repeated failures and resets promptly after recovery. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
271 lines
7.2 KiB
Go
271 lines
7.2 KiB
Go
package client
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/fleetdm/fleet/v4/server/fleet"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestGetConfig(t *testing.T) {
|
|
t.Run(
|
|
"config cache", func(t *testing.T) {
|
|
oc := OrbitClient{}
|
|
oc.configCache.config = &fleet.OrbitConfig{}
|
|
oc.configCache.lastUpdated = time.Now().Add(1 * time.Second)
|
|
config, err := oc.GetConfig()
|
|
require.NoError(t, err)
|
|
require.Equal(t, oc.configCache.config, config)
|
|
},
|
|
)
|
|
t.Run(
|
|
"config cache error", func(t *testing.T) {
|
|
oc := OrbitClient{}
|
|
oc.configCache.config = nil
|
|
oc.configCache.err = errors.New("test error")
|
|
oc.configCache.lastUpdated = time.Now().Add(1 * time.Second)
|
|
config, err := oc.GetConfig()
|
|
require.Error(t, err)
|
|
require.Equal(t, oc.configCache.config, config)
|
|
},
|
|
)
|
|
}
|
|
|
|
func clientWithConfig(cfg *fleet.OrbitConfig) *OrbitClient {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
oc := &OrbitClient{
|
|
receiverUpdateContext: ctx,
|
|
receiverUpdateCancelFunc: cancel,
|
|
}
|
|
oc.configCache.config = cfg
|
|
oc.configCache.lastUpdated = time.Now().Add(1 * time.Hour)
|
|
return oc
|
|
}
|
|
|
|
func TestConfigReceiverCalls(t *testing.T) {
|
|
var called1, called2 bool
|
|
|
|
testmsg := json.RawMessage("testing")
|
|
|
|
rfunc1 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
|
|
if !reflect.DeepEqual(cfg.Flags, testmsg) {
|
|
return errors.New("not equal testmsg")
|
|
}
|
|
called1 = true
|
|
return nil
|
|
})
|
|
rfunc2 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
|
|
if !reflect.DeepEqual(cfg.Flags, testmsg) {
|
|
return errors.New("not equal testmsg")
|
|
}
|
|
called2 = true
|
|
return nil
|
|
})
|
|
|
|
client := clientWithConfig(&fleet.OrbitConfig{Flags: testmsg})
|
|
client.RegisterConfigReceiver(rfunc1)
|
|
client.RegisterConfigReceiver(rfunc2)
|
|
|
|
err := client.RunConfigReceivers()
|
|
require.NoError(t, err)
|
|
|
|
require.True(t, called1)
|
|
require.True(t, called2)
|
|
}
|
|
|
|
func TestConfigReceiverErrors(t *testing.T) {
|
|
var called1, called2 bool
|
|
|
|
rfunc1 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
|
|
called1 = true
|
|
return nil
|
|
})
|
|
rfunc2 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
|
|
called2 = true
|
|
return nil
|
|
})
|
|
err1 := errors.New("error1")
|
|
err2 := errors.New("error2")
|
|
efunc1 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
|
|
return err1
|
|
})
|
|
efunc2 := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
|
|
return err2
|
|
})
|
|
// Make sure we don't get stuck or crash on receiver panic
|
|
pfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
|
|
panic("woah")
|
|
})
|
|
|
|
client := clientWithConfig(&fleet.OrbitConfig{})
|
|
client.RegisterConfigReceiver(efunc1)
|
|
client.RegisterConfigReceiver(rfunc1)
|
|
client.RegisterConfigReceiver(efunc2)
|
|
client.RegisterConfigReceiver(rfunc2)
|
|
client.RegisterConfigReceiver(pfunc)
|
|
|
|
err := client.RunConfigReceivers()
|
|
require.ErrorIs(t, err, err1)
|
|
require.ErrorIs(t, err, err2)
|
|
|
|
require.True(t, called1)
|
|
require.True(t, called2)
|
|
}
|
|
|
|
func TestExecuteConfigReceiversCancel(t *testing.T) {
|
|
client := clientWithConfig(&fleet.OrbitConfig{})
|
|
client.ReceiverUpdateInterval = 100 * time.Millisecond
|
|
|
|
var calls1, calls2 int
|
|
requiredCalls := 4
|
|
|
|
cfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
|
|
calls1++
|
|
if calls1 == requiredCalls {
|
|
client.receiverUpdateCancelFunc()
|
|
}
|
|
return nil
|
|
})
|
|
|
|
rfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
|
|
calls2++
|
|
return nil
|
|
})
|
|
|
|
client.RegisterConfigReceiver(cfunc)
|
|
client.RegisterConfigReceiver(rfunc)
|
|
|
|
err := client.ExecuteConfigReceivers()
|
|
|
|
require.Nil(t, err)
|
|
require.Equal(t, requiredCalls, calls1)
|
|
require.Equal(t, requiredCalls, calls2)
|
|
}
|
|
|
|
func TestExecuteConfigReceiversInterrupt(t *testing.T) {
|
|
client := clientWithConfig(&fleet.OrbitConfig{})
|
|
defer client.receiverUpdateCancelFunc()
|
|
|
|
client.ReceiverUpdateInterval = 100 * time.Millisecond
|
|
|
|
var called bool
|
|
rfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
|
|
called = true
|
|
return nil
|
|
})
|
|
|
|
client.RegisterConfigReceiver(rfunc)
|
|
|
|
finChan := make(chan error)
|
|
go func() {
|
|
finChan <- client.ExecuteConfigReceivers()
|
|
}()
|
|
|
|
go func() {
|
|
time.Sleep(500 * time.Millisecond)
|
|
client.receiverUpdateCancelFunc()
|
|
}()
|
|
|
|
select {
|
|
case err := <-finChan:
|
|
require.Nil(t, err)
|
|
require.True(t, called)
|
|
case <-time.NewTimer(2 * time.Second).C:
|
|
require.Fail(t, "receiver interrupt cancel didn't work")
|
|
}
|
|
}
|
|
|
|
func TestExecuteConfigReceiversBackoffOnError(t *testing.T) {
|
|
client := clientWithConfig(&fleet.OrbitConfig{})
|
|
client.ReceiverUpdateInterval = 1 * time.Second
|
|
|
|
var callTimes []time.Time
|
|
callCount := 0
|
|
// 3 failures then cancel: intervals should be ~1s (base tick), ~2s, ~4s.
|
|
targetCalls := 4
|
|
|
|
rfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
|
|
callTimes = append(callTimes, time.Now())
|
|
callCount++
|
|
if callCount >= targetCalls {
|
|
client.receiverUpdateCancelFunc()
|
|
return nil
|
|
}
|
|
return errors.New("server error")
|
|
})
|
|
|
|
client.RegisterConfigReceiver(rfunc)
|
|
|
|
done := make(chan error, 1)
|
|
go func() { done <- client.ExecuteConfigReceivers() }()
|
|
select {
|
|
case err := <-done:
|
|
require.NoError(t, err)
|
|
case <-time.After(30 * time.Second):
|
|
t.Fatal("test timed out waiting for ExecuteConfigReceivers")
|
|
}
|
|
require.Equal(t, targetCalls, callCount)
|
|
|
|
// Verify each successive interval is strictly longer than the previous.
|
|
// Call 0->1 is the base tick (~1s), 1->2 should be ~2s, 2->3 should be ~4s.
|
|
require.GreaterOrEqual(t, len(callTimes), 3, "need at least 3 calls to verify growth")
|
|
for i := 1; i < len(callTimes)-1; i++ {
|
|
prev := callTimes[i].Sub(callTimes[i-1])
|
|
curr := callTimes[i+1].Sub(callTimes[i])
|
|
assert.Greater(t, curr, prev,
|
|
"interval %d->%d (%v) should be greater than %d->%d (%v)",
|
|
i, i+1, curr, i-1, i, prev)
|
|
}
|
|
}
|
|
|
|
func TestExecuteConfigReceiversResetOnSuccess(t *testing.T) {
|
|
client := clientWithConfig(&fleet.OrbitConfig{})
|
|
client.ReceiverUpdateInterval = 1 * time.Second
|
|
|
|
callCount := 0
|
|
var intervalAfterRecovery time.Duration
|
|
var recoveryStart time.Time
|
|
|
|
rfunc := fleet.OrbitConfigReceiverFunc(func(cfg *fleet.OrbitConfig) error {
|
|
callCount++
|
|
switch {
|
|
case callCount <= 2:
|
|
// First 2 calls fail -- build up backoff
|
|
return errors.New("server error")
|
|
case callCount == 3:
|
|
// Success -- should reset backoff
|
|
recoveryStart = time.Now()
|
|
return nil
|
|
case callCount == 4:
|
|
// Next call should be at base interval (~1s), not backed off
|
|
intervalAfterRecovery = time.Since(recoveryStart)
|
|
client.receiverUpdateCancelFunc()
|
|
return nil
|
|
}
|
|
return nil
|
|
})
|
|
|
|
client.RegisterConfigReceiver(rfunc)
|
|
|
|
done := make(chan error, 1)
|
|
go func() { done <- client.ExecuteConfigReceivers() }()
|
|
select {
|
|
case err := <-done:
|
|
require.NoError(t, err)
|
|
case <-time.After(30 * time.Second):
|
|
t.Fatal("test timed out waiting for ExecuteConfigReceivers")
|
|
}
|
|
require.Equal(t, 4, callCount)
|
|
|
|
// After recovery, interval should be close to base (1s), not backed off.
|
|
// Use 2s as the upper bound: base (1s) + jitter (up to 10%) + scheduling slack.
|
|
assert.Less(t, intervalAfterRecovery, 2*time.Second,
|
|
"after success, interval should reset near base, got %v", intervalAfterRecovery)
|
|
}
|