Add self-heal mechanism in case of corruption in osquery or Fleet Desktop executables (#47818)

Resolves #47552 Currently, a corruption in the download process is caught by our TUF updater and will re-download. So the main scenario we are covering here is a corruption in the extraction process of the .tar.gz components. I'm simulating this by modifying the executables in the hosts and restarting (now with these changes it self-heals). - [X] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. ## Testing - [X] Added/updated automated tests - [x] QA'd all new/changed functionality manually ## fleetd/orbit/Fleet Desktop - [x] Verified compatibility with the latest released version of Fleet (see [Must rule](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/workflows/fleetd-development-and-release-strategy.md)) - [x] If the change applies to only one platform, confirmed that `runtime.GOOS` is used as needed to isolate changes - [x] Verified that fleetd runs on macOS, Linux and Windows - [x] Verified auto-update works from the released version of component to the new version (see [tools/tuf/test](../tools/tuf/test/README.md))  ## Summary by CodeRabbit * **New Features** * Orbit now self-heals corrupt component binaries by detecting executables that fail to run, removing the broken artifacts, re-downloading, and re-verifying before continuing (including the osqueryd and Fleet Desktop components). * **Bug Fixes** * Prevents endless crash loops caused by truncated or otherwise invalid on-disk binaries. * **Tests** * Added coverage for exec verification and target cleanup/re-download, including corruption, healthy binaries, and cross-platform behavior.
2026-06-18 10:08:35 -03:00
parent b28e6ceaaf
commit 443d82dd15
4 changed files with 261 additions and 2 deletions
@@ -0,0 +1 @@
+- Fixed an issue where a corrupt osqueryd or Fleet Desktop binary (e.g. from a truncated update download) would cause orbit to crash-loop indefinitely. Orbit now detects an executable that fails to run, removes the corrupt component, and re-downloads it from the update server.
@@ -1701,6 +1701,49 @@ func setServerOverrides(c *cli.Context) fallbackServerOverridesConfig {
 	return overrideCfg.fallbackServerOverridesConfig
 }

+// getComponentWithSelfHeal resolves a component target via the updater and
+// self-heals from a corrupt binary.
+//
+// After downloading/locating the target it verifies the installed executable
+// can run (CheckExec runs it with --help, or the target's CustomCheckExec). If
+// it fails to run for any reason (e.g. a truncated TUF download/extraction that
+// fails to fork/exec or execs and crashes), it removes the on-disk artifacts,
+// re-downloads from TUF, and re-verifies. Without this, orbit records the bad
+// path as last-known-good and crash-loops on it forever (see
+// https://github.com/fleetdm/fleet/issues/47552).
+func getComponentWithSelfHeal(updater *update.Updater, target string) (*update.LocalTarget, error) {
+	localTarget, err := updater.Get(target)
+	if err != nil {
+		return nil, err
+	}
+
+	checkErr := updater.CheckExec(target)
+	if checkErr == nil {
+		return localTarget, nil
+	}
+
+	// The installed binary failed to run (e.g. a truncated TUF
+	// download/extraction that fails to fork/exec, or that execs and then
+	// crashes). Whatever the cause, it's unusable, so remove the on-disk
+	// artifacts, re-download from TUF, and re-verify. Self-heal is a single
+	// attempt: if the re-downloaded binary still fails the exec check we return
+	// the error rather than crash-looping on it forever.
+	log.Error().Err(checkErr).Str("target", target).Msg("component binary failed exec check, self-healing")
+	if err := updater.RemoveTarget(target); err != nil {
+		return nil, fmt.Errorf("self-heal remove %s: %w", target, err)
+	}
+	localTarget, err = updater.Get(target)
+	if err != nil {
+		return nil, fmt.Errorf("self-heal re-download %s: %w", target, err)
+	}
+	if err := updater.CheckExec(target); err != nil {
+		return nil, fmt.Errorf("%s still failing exec check after self-heal: %w", target, err)
+	}
+	log.Info().Str("target", target).Msg("component self-heal succeeded")
+
+	return localTarget, nil
+}
+
 // getFleetdComponentPaths returns the paths of the fleetd components.
 // If the path to the component cannot be fetched using the updater (e.g. channel doesn't exist yet)
 // then it will use the fallbackCfg's paths (if set).
@@ -1761,7 +1804,7 @@ func getFleetdComponentPaths(
 	}

 	// osqueryd
-	osquerydLocalTarget, err := updater.Get(constant.OsqueryTUFTargetName)
+	osquerydLocalTarget, err := getComponentWithSelfHeal(updater, constant.OsqueryTUFTargetName)
 	if err != nil {
 		if fallbackCfg.OsquerydPath == "" {
 			log.Info().Err(err).Msgf("get %s target failed", constant.OsqueryTUFTargetName)
@@ -1775,7 +1818,7 @@ func getFleetdComponentPaths(

 	// Fleet Desktop
 	if c.Bool("fleet-desktop") {
-		fleetDesktopLocalTarget, err := updater.Get(constant.DesktopTUFTargetName)
+		fleetDesktopLocalTarget, err := getComponentWithSelfHeal(updater, constant.DesktopTUFTargetName)
 		if err != nil {
 			if fallbackCfg.DesktopPath == "" {
 				log.Info().Err(err).Msgf("get %s target failed", constant.DesktopTUFTargetName)
@@ -0,0 +1,82 @@
+package update
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+
+	"github.com/rs/zerolog/log"
+)
+
+// CheckExec verifies that the target's installed executable can run, using the
+// same check applied to freshly downloaded targets (the target's CustomCheckExec
+// if set, otherwise running it with --help).
+//
+// A non-nil error means the on-disk executable failed to run (corrupt/truncated
+// download, crash on startup, etc.) and the caller should self-heal by
+// re-downloading it.
+//
+// Unlike the download-path checkExec, this needs no platform/arch guards: it
+// only runs in orbit, which loads targets matching the host OS/arch.
+func (u *Updater) CheckExec(target string) error {
+	localTarget, err := u.localTarget(target)
+	if err != nil {
+		return fmt.Errorf("load local target %s: %w", target, err)
+	}
+
+	if localTarget.Info.CustomCheckExec != nil {
+		if err := localTarget.Info.CustomCheckExec(localTarget.ExecPath); err != nil {
+			return fmt.Errorf("custom exec check %q: %w", localTarget.ExecPath, err)
+		}
+		return nil
+	}
+
+	// Note: this would fail for any binary that returns nonzero for --help.
+	cmd := exec.Command(localTarget.ExecPath, "--help")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("exec check %q: %s: %w", localTarget.ExecPath, string(out), err)
+	}
+	return nil
+}
+
+// RemoveTarget removes the on-disk artifacts for the given target so that the
+// next call to Get re-downloads and re-extracts it from the remote TUF
+// repository. It removes:
+//
+//   - the extracted directory (e.g. .../<version>/osquery.app), if any;
+//   - the downloaded archive (e.g. .../osqueryd.app.tar.gz); and
+//   - the cached archive hash (.sha512).
+//
+// Removing the archive (not just the extracted directory) forces a fresh
+// download from TUF rather than re-extracting a possibly-corrupt archive.
+//
+// This is used to self-heal from a component binary that fails its exec check
+// (a corrupt/truncated download that won't fork/exec or crashes on startup).
+func (u *Updater) RemoveTarget(target string) error {
+	localTarget, err := u.localTarget(target)
+	if err != nil {
+		return fmt.Errorf("load local target %s: %w", target, err)
+	}
+
+	// Remove the extracted directory (e.g. .../<version>/osquery.app), if any.
+	if localTarget.DirPath != "" {
+		if err := os.RemoveAll(localTarget.DirPath); err != nil {
+			return fmt.Errorf("remove extracted dir %q: %w", localTarget.DirPath, err)
+		}
+	}
+
+	// Remove the downloaded archive and its cached hash so the next Get
+	// re-downloads from TUF instead of re-extracting a possibly-corrupt archive.
+	if err := os.RemoveAll(localTarget.Path); err != nil {
+		return fmt.Errorf("remove archive %q: %w", localTarget.Path, err)
+	}
+	removeCachedHashes(localTarget.Path)
+
+	log.Info().
+		Str("target", target).
+		Str("path", localTarget.Path).
+		Str("dir", localTarget.DirPath).
+		Msg("removed corrupt target for re-download")
+
+	return nil
+}
@@ -0,0 +1,133 @@
+package update
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRemoveTarget(t *testing.T) {
+	t.Parallel()
+
+	tmpDir := t.TempDir()
+	const target = "osqueryd"
+	info := TargetInfo{
+		Platform:             "macos-app",
+		Channel:              "5.22.1",
+		TargetFile:           "osqueryd.app.tar.gz",
+		ExtractedExecSubPath: []string{"osquery.app", "Contents", "MacOS", "osqueryd"},
+	}
+	u := &Updater{opt: Options{
+		RootDirectory: tmpDir,
+		Targets:       Targets{target: info},
+	}}
+
+	archivePath, execPath, dirPath := LocalTargetPaths(tmpDir, target, info)
+	hashPath := archivePath + ".sha512"
+
+	// Lay down the archive, cached hash and the extracted (corrupt) binary.
+	require.NoError(t, os.MkdirAll(filepath.Dir(archivePath), 0o755))
+	require.NoError(t, os.MkdirAll(filepath.Dir(execPath), 0o755))
+	require.NoError(t, os.WriteFile(archivePath, []byte("archive"), 0o644))
+	require.NoError(t, os.WriteFile(hashPath, []byte("deadbeef"), 0o644))
+	require.NoError(t, os.WriteFile(execPath, []byte("truncated"), 0o755)) // #nosec G306
+
+	require.NoError(t, u.RemoveTarget(target))
+
+	// All three artifacts should be gone so the next Get re-downloads.
+	for _, p := range []string{archivePath, hashPath, dirPath, execPath} {
+		_, err := os.Stat(p)
+		require.ErrorIs(t, err, os.ErrNotExist, "expected %q removed", p)
+	}
+}
+
+func TestCheckExec(t *testing.T) {
+	t.Parallel()
+
+	platform := map[string]string{
+		"darwin":  "macos",
+		"linux":   "linux",
+		"windows": "windows",
+	}[runtime.GOOS]
+	require.NotEmpty(t, platform, "unsupported test platform %s", runtime.GOOS)
+
+	const target = "osqueryd"
+
+	t.Run("corrupt binary surfaces a corruption error", func(t *testing.T) {
+		u := &Updater{opt: Options{
+			RootDirectory: t.TempDir(),
+			Targets: Targets{target: TargetInfo{
+				Platform:   platform,
+				Channel:    "stable",
+				TargetFile: "osqueryd",
+				CustomCheckExec: func(string) error {
+					return fmt.Errorf("fork/exec: %w", syscall.ENOEXEC)
+				},
+			}},
+		}}
+		err := u.CheckExec(target)
+		require.Error(t, err)
+	})
+
+	t.Run("healthy binary passes", func(t *testing.T) {
+		u := &Updater{opt: Options{
+			RootDirectory: t.TempDir(),
+			Targets: Targets{target: TargetInfo{
+				Platform:        platform,
+				Channel:         "stable",
+				TargetFile:      "osqueryd",
+				CustomCheckExec: func(string) error { return nil },
+			}},
+		}}
+		require.NoError(t, u.CheckExec(target))
+	})
+}
+
+// TestCheckExecRealBinary exercises the default `--help` exec branch (no
+// CustomCheckExec) against real files on disk. This is the branch osqueryd
+// actually uses, so it must run an actual executable rather than a stub.
+func TestCheckExecRealBinary(t *testing.T) {
+	t.Parallel()
+
+	platform := map[string]string{
+		"darwin":  "macos",
+		"linux":   "linux",
+		"windows": "windows",
+	}[runtime.GOOS]
+	require.NotEmpty(t, platform, "unsupported test platform %s", runtime.GOOS)
+
+	const target = "osqueryd"
+	info := TargetInfo{
+		Platform:   platform,
+		Channel:    "stable",
+		TargetFile: "osqueryd",
+	}
+	root := t.TempDir()
+	u := &Updater{opt: Options{RootDirectory: root, Targets: Targets{target: info}}}
+
+	_, execPath, _ := LocalTargetPaths(root, target, info)
+	require.NoError(t, os.MkdirAll(filepath.Dir(execPath), 0o755))
+
+	t.Run("healthy binary passes --help", func(t *testing.T) {
+		if runtime.GOOS == "windows" {
+			t.Skip("the shell-script stand-in binary is unix-only")
+		}
+		// A script that exits 0 for any args (including --help).
+		require.NoError(t, os.WriteFile(execPath, []byte("#!/bin/sh\nexit 0\n"), 0o755)) // #nosec G306
+		require.NoError(t, u.CheckExec(target))
+	})
+
+	t.Run("corrupt binary fails the exec check", func(t *testing.T) {
+		// Non-executable garbage (no shebang, not a valid Mach-O/ELF/PE) fails to
+		// fork/exec with a format error on every platform: ENOEXEC ("exec format
+		// error") on Linux/macOS, ERROR_BAD_EXE_FORMAT on Windows.
+		require.NoError(t, os.WriteFile(execPath, []byte("\x00\x01\x02not a binary"), 0o755)) // #nosec G306
+		err := u.CheckExec(target)
+		require.Error(t, err)
+	})
+}
				`@@ -0,0 +1 @@`
				`- Fixed an issue where a corrupt osqueryd or Fleet Desktop binary (e.g. from a truncated update download) would cause orbit to crash-loop indefinitely. Orbit now detects an executable that fails to run, removes the corrupt component, and re-downloads it from the update server.`