Files
fleet/orbit/pkg/update/selfheal.go
T
Lucas Manuel Rodriguez 443d82dd15 Add self-heal mechanism in case of corruption in osquery or Fleet Desktop executables (#47818)
Resolves #47552

Currently, a corruption in the download process is caught by our TUF
updater and will re-download.
So the main scenario we are covering here is a corruption in the
extraction process of the .tar.gz components.
I'm simulating this by modifying the executables in the hosts and
restarting (now with these changes it self-heals).

- [X] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.

## Testing

- [X] Added/updated automated tests
- [x] QA'd all new/changed functionality manually

## fleetd/orbit/Fleet Desktop

- [x] Verified compatibility with the latest released version of Fleet
(see [Must
rule](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/workflows/fleetd-development-and-release-strategy.md))
- [x] If the change applies to only one platform, confirmed that
`runtime.GOOS` is used as needed to isolate changes
- [x] Verified that fleetd runs on macOS, Linux and Windows
- [x] Verified auto-update works from the released version of component
to the new version (see [tools/tuf/test](../tools/tuf/test/README.md))


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Orbit now self-heals corrupt component binaries by detecting
executables that fail to run, removing the broken artifacts,
re-downloading, and re-verifying before continuing (including the
osqueryd and Fleet Desktop components).

* **Bug Fixes**
* Prevents endless crash loops caused by truncated or otherwise invalid
on-disk binaries.

* **Tests**
* Added coverage for exec verification and target cleanup/re-download,
including corruption, healthy binaries, and cross-platform behavior.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
2026-06-18 10:08:35 -03:00

83 lines
2.8 KiB
Go

package update
import (
"fmt"
"os"
"os/exec"
"github.com/rs/zerolog/log"
)
// CheckExec verifies that the target's installed executable can run, using the
// same check applied to freshly downloaded targets (the target's CustomCheckExec
// if set, otherwise running it with --help).
//
// A non-nil error means the on-disk executable failed to run (corrupt/truncated
// download, crash on startup, etc.) and the caller should self-heal by
// re-downloading it.
//
// Unlike the download-path checkExec, this needs no platform/arch guards: it
// only runs in orbit, which loads targets matching the host OS/arch.
func (u *Updater) CheckExec(target string) error {
localTarget, err := u.localTarget(target)
if err != nil {
return fmt.Errorf("load local target %s: %w", target, err)
}
if localTarget.Info.CustomCheckExec != nil {
if err := localTarget.Info.CustomCheckExec(localTarget.ExecPath); err != nil {
return fmt.Errorf("custom exec check %q: %w", localTarget.ExecPath, err)
}
return nil
}
// Note: this would fail for any binary that returns nonzero for --help.
cmd := exec.Command(localTarget.ExecPath, "--help")
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("exec check %q: %s: %w", localTarget.ExecPath, string(out), err)
}
return nil
}
// RemoveTarget removes the on-disk artifacts for the given target so that the
// next call to Get re-downloads and re-extracts it from the remote TUF
// repository. It removes:
//
// - the extracted directory (e.g. .../<version>/osquery.app), if any;
// - the downloaded archive (e.g. .../osqueryd.app.tar.gz); and
// - the cached archive hash (.sha512).
//
// Removing the archive (not just the extracted directory) forces a fresh
// download from TUF rather than re-extracting a possibly-corrupt archive.
//
// This is used to self-heal from a component binary that fails its exec check
// (a corrupt/truncated download that won't fork/exec or crashes on startup).
func (u *Updater) RemoveTarget(target string) error {
localTarget, err := u.localTarget(target)
if err != nil {
return fmt.Errorf("load local target %s: %w", target, err)
}
// Remove the extracted directory (e.g. .../<version>/osquery.app), if any.
if localTarget.DirPath != "" {
if err := os.RemoveAll(localTarget.DirPath); err != nil {
return fmt.Errorf("remove extracted dir %q: %w", localTarget.DirPath, err)
}
}
// Remove the downloaded archive and its cached hash so the next Get
// re-downloads from TUF instead of re-extracting a possibly-corrupt archive.
if err := os.RemoveAll(localTarget.Path); err != nil {
return fmt.Errorf("remove archive %q: %w", localTarget.Path, err)
}
removeCachedHashes(localTarget.Path)
log.Info().
Str("target", target).
Str("path", localTarget.Path).
Str("dir", localTarget.DirPath).
Msg("removed corrupt target for re-download")
return nil
}