From 443d82dd15edc3210e0fd0abe59cf3de2efc50f0 Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Thu, 18 Jun 2026 10:08:35 -0300 Subject: [PATCH] Add self-heal mechanism in case of corruption in osquery or Fleet Desktop executables (#47818) Resolves #47552 Currently, a corruption in the download process is caught by our TUF updater and will re-download. So the main scenario we are covering here is a corruption in the extraction process of the .tar.gz components. I'm simulating this by modifying the executables in the hosts and restarting (now with these changes it self-heals). - [X] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. ## Testing - [X] Added/updated automated tests - [x] QA'd all new/changed functionality manually ## fleetd/orbit/Fleet Desktop - [x] Verified compatibility with the latest released version of Fleet (see [Must rule](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/workflows/fleetd-development-and-release-strategy.md)) - [x] If the change applies to only one platform, confirmed that `runtime.GOOS` is used as needed to isolate changes - [x] Verified that fleetd runs on macOS, Linux and Windows - [x] Verified auto-update works from the released version of component to the new version (see [tools/tuf/test](../tools/tuf/test/README.md)) ## Summary by CodeRabbit * **New Features** * Orbit now self-heals corrupt component binaries by detecting executables that fail to run, removing the broken artifacts, re-downloading, and re-verifying before continuing (including the osqueryd and Fleet Desktop components). * **Bug Fixes** * Prevents endless crash loops caused by truncated or otherwise invalid on-disk binaries. * **Tests** * Added coverage for exec verification and target cleanup/re-download, including corruption, healthy binaries, and cross-platform behavior. --- .../47552-orbit-self-heal-corrupt-binary | 1 + orbit/cmd/orbit/orbit.go | 47 ++++++- orbit/pkg/update/selfheal.go | 82 +++++++++++ orbit/pkg/update/selfheal_test.go | 133 ++++++++++++++++++ 4 files changed, 261 insertions(+), 2 deletions(-) create mode 100644 orbit/changes/47552-orbit-self-heal-corrupt-binary create mode 100644 orbit/pkg/update/selfheal.go create mode 100644 orbit/pkg/update/selfheal_test.go diff --git a/orbit/changes/47552-orbit-self-heal-corrupt-binary b/orbit/changes/47552-orbit-self-heal-corrupt-binary new file mode 100644 index 0000000000..c99306567f --- /dev/null +++ b/orbit/changes/47552-orbit-self-heal-corrupt-binary @@ -0,0 +1 @@ +- Fixed an issue where a corrupt osqueryd or Fleet Desktop binary (e.g. from a truncated update download) would cause orbit to crash-loop indefinitely. Orbit now detects an executable that fails to run, removes the corrupt component, and re-downloads it from the update server. diff --git a/orbit/cmd/orbit/orbit.go b/orbit/cmd/orbit/orbit.go index feec75cdfa..7eea26a9d8 100644 --- a/orbit/cmd/orbit/orbit.go +++ b/orbit/cmd/orbit/orbit.go @@ -1701,6 +1701,49 @@ func setServerOverrides(c *cli.Context) fallbackServerOverridesConfig { return overrideCfg.fallbackServerOverridesConfig } +// getComponentWithSelfHeal resolves a component target via the updater and +// self-heals from a corrupt binary. +// +// After downloading/locating the target it verifies the installed executable +// can run (CheckExec runs it with --help, or the target's CustomCheckExec). If +// it fails to run for any reason (e.g. a truncated TUF download/extraction that +// fails to fork/exec or execs and crashes), it removes the on-disk artifacts, +// re-downloads from TUF, and re-verifies. Without this, orbit records the bad +// path as last-known-good and crash-loops on it forever (see +// https://github.com/fleetdm/fleet/issues/47552). +func getComponentWithSelfHeal(updater *update.Updater, target string) (*update.LocalTarget, error) { + localTarget, err := updater.Get(target) + if err != nil { + return nil, err + } + + checkErr := updater.CheckExec(target) + if checkErr == nil { + return localTarget, nil + } + + // The installed binary failed to run (e.g. a truncated TUF + // download/extraction that fails to fork/exec, or that execs and then + // crashes). Whatever the cause, it's unusable, so remove the on-disk + // artifacts, re-download from TUF, and re-verify. Self-heal is a single + // attempt: if the re-downloaded binary still fails the exec check we return + // the error rather than crash-looping on it forever. + log.Error().Err(checkErr).Str("target", target).Msg("component binary failed exec check, self-healing") + if err := updater.RemoveTarget(target); err != nil { + return nil, fmt.Errorf("self-heal remove %s: %w", target, err) + } + localTarget, err = updater.Get(target) + if err != nil { + return nil, fmt.Errorf("self-heal re-download %s: %w", target, err) + } + if err := updater.CheckExec(target); err != nil { + return nil, fmt.Errorf("%s still failing exec check after self-heal: %w", target, err) + } + log.Info().Str("target", target).Msg("component self-heal succeeded") + + return localTarget, nil +} + // getFleetdComponentPaths returns the paths of the fleetd components. // If the path to the component cannot be fetched using the updater (e.g. channel doesn't exist yet) // then it will use the fallbackCfg's paths (if set). @@ -1761,7 +1804,7 @@ func getFleetdComponentPaths( } // osqueryd - osquerydLocalTarget, err := updater.Get(constant.OsqueryTUFTargetName) + osquerydLocalTarget, err := getComponentWithSelfHeal(updater, constant.OsqueryTUFTargetName) if err != nil { if fallbackCfg.OsquerydPath == "" { log.Info().Err(err).Msgf("get %s target failed", constant.OsqueryTUFTargetName) @@ -1775,7 +1818,7 @@ func getFleetdComponentPaths( // Fleet Desktop if c.Bool("fleet-desktop") { - fleetDesktopLocalTarget, err := updater.Get(constant.DesktopTUFTargetName) + fleetDesktopLocalTarget, err := getComponentWithSelfHeal(updater, constant.DesktopTUFTargetName) if err != nil { if fallbackCfg.DesktopPath == "" { log.Info().Err(err).Msgf("get %s target failed", constant.DesktopTUFTargetName) diff --git a/orbit/pkg/update/selfheal.go b/orbit/pkg/update/selfheal.go new file mode 100644 index 0000000000..a989b8bcf1 --- /dev/null +++ b/orbit/pkg/update/selfheal.go @@ -0,0 +1,82 @@ +package update + +import ( + "fmt" + "os" + "os/exec" + + "github.com/rs/zerolog/log" +) + +// CheckExec verifies that the target's installed executable can run, using the +// same check applied to freshly downloaded targets (the target's CustomCheckExec +// if set, otherwise running it with --help). +// +// A non-nil error means the on-disk executable failed to run (corrupt/truncated +// download, crash on startup, etc.) and the caller should self-heal by +// re-downloading it. +// +// Unlike the download-path checkExec, this needs no platform/arch guards: it +// only runs in orbit, which loads targets matching the host OS/arch. +func (u *Updater) CheckExec(target string) error { + localTarget, err := u.localTarget(target) + if err != nil { + return fmt.Errorf("load local target %s: %w", target, err) + } + + if localTarget.Info.CustomCheckExec != nil { + if err := localTarget.Info.CustomCheckExec(localTarget.ExecPath); err != nil { + return fmt.Errorf("custom exec check %q: %w", localTarget.ExecPath, err) + } + return nil + } + + // Note: this would fail for any binary that returns nonzero for --help. + cmd := exec.Command(localTarget.ExecPath, "--help") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("exec check %q: %s: %w", localTarget.ExecPath, string(out), err) + } + return nil +} + +// RemoveTarget removes the on-disk artifacts for the given target so that the +// next call to Get re-downloads and re-extracts it from the remote TUF +// repository. It removes: +// +// - the extracted directory (e.g. ...//osquery.app), if any; +// - the downloaded archive (e.g. .../osqueryd.app.tar.gz); and +// - the cached archive hash (.sha512). +// +// Removing the archive (not just the extracted directory) forces a fresh +// download from TUF rather than re-extracting a possibly-corrupt archive. +// +// This is used to self-heal from a component binary that fails its exec check +// (a corrupt/truncated download that won't fork/exec or crashes on startup). +func (u *Updater) RemoveTarget(target string) error { + localTarget, err := u.localTarget(target) + if err != nil { + return fmt.Errorf("load local target %s: %w", target, err) + } + + // Remove the extracted directory (e.g. ...//osquery.app), if any. + if localTarget.DirPath != "" { + if err := os.RemoveAll(localTarget.DirPath); err != nil { + return fmt.Errorf("remove extracted dir %q: %w", localTarget.DirPath, err) + } + } + + // Remove the downloaded archive and its cached hash so the next Get + // re-downloads from TUF instead of re-extracting a possibly-corrupt archive. + if err := os.RemoveAll(localTarget.Path); err != nil { + return fmt.Errorf("remove archive %q: %w", localTarget.Path, err) + } + removeCachedHashes(localTarget.Path) + + log.Info(). + Str("target", target). + Str("path", localTarget.Path). + Str("dir", localTarget.DirPath). + Msg("removed corrupt target for re-download") + + return nil +} diff --git a/orbit/pkg/update/selfheal_test.go b/orbit/pkg/update/selfheal_test.go new file mode 100644 index 0000000000..b984122e8e --- /dev/null +++ b/orbit/pkg/update/selfheal_test.go @@ -0,0 +1,133 @@ +package update + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "syscall" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRemoveTarget(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + const target = "osqueryd" + info := TargetInfo{ + Platform: "macos-app", + Channel: "5.22.1", + TargetFile: "osqueryd.app.tar.gz", + ExtractedExecSubPath: []string{"osquery.app", "Contents", "MacOS", "osqueryd"}, + } + u := &Updater{opt: Options{ + RootDirectory: tmpDir, + Targets: Targets{target: info}, + }} + + archivePath, execPath, dirPath := LocalTargetPaths(tmpDir, target, info) + hashPath := archivePath + ".sha512" + + // Lay down the archive, cached hash and the extracted (corrupt) binary. + require.NoError(t, os.MkdirAll(filepath.Dir(archivePath), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Dir(execPath), 0o755)) + require.NoError(t, os.WriteFile(archivePath, []byte("archive"), 0o644)) + require.NoError(t, os.WriteFile(hashPath, []byte("deadbeef"), 0o644)) + require.NoError(t, os.WriteFile(execPath, []byte("truncated"), 0o755)) // #nosec G306 + + require.NoError(t, u.RemoveTarget(target)) + + // All three artifacts should be gone so the next Get re-downloads. + for _, p := range []string{archivePath, hashPath, dirPath, execPath} { + _, err := os.Stat(p) + require.ErrorIs(t, err, os.ErrNotExist, "expected %q removed", p) + } +} + +func TestCheckExec(t *testing.T) { + t.Parallel() + + platform := map[string]string{ + "darwin": "macos", + "linux": "linux", + "windows": "windows", + }[runtime.GOOS] + require.NotEmpty(t, platform, "unsupported test platform %s", runtime.GOOS) + + const target = "osqueryd" + + t.Run("corrupt binary surfaces a corruption error", func(t *testing.T) { + u := &Updater{opt: Options{ + RootDirectory: t.TempDir(), + Targets: Targets{target: TargetInfo{ + Platform: platform, + Channel: "stable", + TargetFile: "osqueryd", + CustomCheckExec: func(string) error { + return fmt.Errorf("fork/exec: %w", syscall.ENOEXEC) + }, + }}, + }} + err := u.CheckExec(target) + require.Error(t, err) + }) + + t.Run("healthy binary passes", func(t *testing.T) { + u := &Updater{opt: Options{ + RootDirectory: t.TempDir(), + Targets: Targets{target: TargetInfo{ + Platform: platform, + Channel: "stable", + TargetFile: "osqueryd", + CustomCheckExec: func(string) error { return nil }, + }}, + }} + require.NoError(t, u.CheckExec(target)) + }) +} + +// TestCheckExecRealBinary exercises the default `--help` exec branch (no +// CustomCheckExec) against real files on disk. This is the branch osqueryd +// actually uses, so it must run an actual executable rather than a stub. +func TestCheckExecRealBinary(t *testing.T) { + t.Parallel() + + platform := map[string]string{ + "darwin": "macos", + "linux": "linux", + "windows": "windows", + }[runtime.GOOS] + require.NotEmpty(t, platform, "unsupported test platform %s", runtime.GOOS) + + const target = "osqueryd" + info := TargetInfo{ + Platform: platform, + Channel: "stable", + TargetFile: "osqueryd", + } + root := t.TempDir() + u := &Updater{opt: Options{RootDirectory: root, Targets: Targets{target: info}}} + + _, execPath, _ := LocalTargetPaths(root, target, info) + require.NoError(t, os.MkdirAll(filepath.Dir(execPath), 0o755)) + + t.Run("healthy binary passes --help", func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the shell-script stand-in binary is unix-only") + } + // A script that exits 0 for any args (including --help). + require.NoError(t, os.WriteFile(execPath, []byte("#!/bin/sh\nexit 0\n"), 0o755)) // #nosec G306 + require.NoError(t, u.CheckExec(target)) + }) + + t.Run("corrupt binary fails the exec check", func(t *testing.T) { + // Non-executable garbage (no shebang, not a valid Mach-O/ELF/PE) fails to + // fork/exec with a format error on every platform: ENOEXEC ("exec format + // error") on Linux/macOS, ERROR_BAD_EXE_FORMAT on Windows. + require.NoError(t, os.WriteFile(execPath, []byte("\x00\x01\x02not a binary"), 0o755)) // #nosec G306 + err := u.CheckExec(target) + require.Error(t, err) + }) +}