Files
fleet/orbit/pkg/execuser/execuser_linux.go
T
Nico 0ac698145b Reap sudo child processes in execuser on Linux to prevent zombies (#47664)
<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #41796

Root cause: run() in orbit/pkg/execuser/execuser_linux.go called
cmd.Start() but never cmd.Wait(). On Linux a child that exits stays a
zombie until the parent reaps it, so every sudo … fleet-desktop
invocation leaked a zombie. When Fleet Desktop fails to start, orbit
respawns it in a loop (desktopRunner.Execute), so the zombies pile up.
 
Fix: reap the child in a background goroutine after Start(). run() still
returns immediately (orbit monitors the desktop process separately), so
behavior is unchanged otherwise.

# Checklist for submitter

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

## Testing

- [x] QA'd all new/changed functionality manually

#### Reproduction steps

1. Start TUF server and create .deb package
```sh
SYSTEMS="linux-arm64" \
DEB_FLEET_URL=https://nicolasulmete.xyz \  # personal ngrok tunnel serving Fleet
DEB_TUF_URL=http://nicotuf.ngrok.io \            # personal ngrok tunnel serving TUF
GENERATE_DEB_ARM64=1 \
ENROLL_SECRET=<enroll-secret> \
FLEET_DESKTOP=1 \
TUF_PATH=$FLEET_REPO_DIR/test_tuf \
./tools/tuf/test/main.sh
```

2. Install .deb on a Ubuntu 25.10 ARM64 VM.
3. Push a failing fleet-desktop stub to trigger the respawn loop that
produces zombie processes:

```sh
export TUF_PATH=$FLEET_REPO_DIR/test_tuf
export FLEET_TARGETS_PASSPHRASE=p4ssphr4s3 FLEET_SNAPSHOT_PASSPHRASE=p4ssphr4s3 FLEET_TIMESTAMP_PASSPHRASE=p4ssphr4s3
source ./tools/tuf/test/load_orbit_version_vars.sh
STUB=$(mktemp -d)
mkdir -p "$STUB/fleet-desktop"
printf '#!/bin/sh\n[ "$1" = "--help" ] && exit 0\nexit 1\n' > "$STUB/fleet-desktop/fleet-desktop"
chmod +x "$STUB/fleet-desktop/fleet-desktop"
tar czf "$STUB/desktop.tar.gz" -C "$STUB" fleet-desktop
./build/fleetctl updates add --path "$TUF_PATH" --target "$STUB/desktop.tar.gz" --platform linux-arm64 --name desktop --version "$ORBIT_VERSION" -t "$ORBIT_MAJOR.$ORBIT_MINOR" -t "$ORBIT_MAJOR" -t stable
rm -rf "$STUB"
```

4. Watch zombie processes on the VM:

```sh
 watch -n5 'ORB=$(pgrep -x orbit); \
    echo "zombie sudo children: $(ps -eo ppid,stat,comm | awk -v o="$ORB" "\$1==o && \$2 ~ /Z/ && \$3==\"sudo\"" | wc -l)"; \
    echo "total sudo lines:     $(ps aux | grep -c "[s]udo")"'
```


https://github.com/user-attachments/assets/dfa4d766-3c8e-4eb0-9104-7219c421538f

#### Test fix

1. Switch to this branch, rebuild and push the new orbit binary (note
that we still have the stub fleet-desktop that always exits).

```sh
make build
export TUF_PATH=$FLEET_REPO_DIR/test_tuf
bash $FLEET_REPO_DIR/push-orbit-arm64.sh
```

Contents of `push-orbit-arm64.sh` is:

```sh
#!/bin/bash
set -e

source ./tools/tuf/test/load_orbit_version_vars.sh

GOOS=linux GOARCH=arm64 go build \
    -o orbit-linux-arm64 \
    -ldflags="-s -w -X github.com/fleetdm/fleet/v4/orbit/pkg/build.Version=$ORBIT_VERSION -X github.com/fleetdm/fleet/v4/orbit/pkg/build.Commit=$ORBIT_COMMIT" \
    ./orbit/cmd/orbit

./tools/tuf/test/push_target.sh linux-arm64 orbit orbit-linux-arm64 "$ORBIT_VERSION"
```

2. Verify on the VM that the update is detected and applied. Also, run
step 4's cmd from the reproduction steps and see the zombies being 0,
even though fleet-desktop is still continuously exiting.


https://github.com/user-attachments/assets/d0f96d85-5e94-4878-b413-67031add3c1e

## fleetd/orbit/Fleet Desktop

- [x] Verified compatibility with the latest released version of Fleet
(see [Must
rule](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/workflows/fleetd-development-and-release-strategy.md))
- [x] If the change applies to only one platform, confirmed that
`runtime.GOOS` is used as needed to isolate changes
- [x] Verified that fleetd runs on macOS, Linux and Windows
- [x] Verified auto-update works from the released version of component
to the new version (see [tools/tuf/test](../tools/tuf/test/README.md))


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **Bug Fixes**
* Fixed an issue on Linux where repeated Fleet Desktop startup failures
could result in thousands of orphaned processes accumulating in the
background, degrading system performance.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
2026-06-16 11:40:52 -03:00

348 lines
11 KiB
Go

package execuser
import (
"bytes"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
userpkg "github.com/fleetdm/fleet/v4/orbit/pkg/user"
"github.com/rs/zerolog/log"
)
// base command to setup an exec.Cmd using `runuser`
func baserun(path string, opts eopts) (cmd *exec.Cmd, err error) {
if opts.user == "" {
return nil, errors.New("missing user")
}
args, env, err := getConfigForCommand(opts.user, path)
if err != nil {
return nil, fmt.Errorf("get args: %w", err)
}
env = append(env,
// Append the packaged libayatana-appindicator3 libraries path to LD_LIBRARY_PATH.
//
// Fleet Desktop doesn't use libayatana-appindicator3 since 1.18.3, but we need to
// keep this to support older versions of Fleet Desktop.
fmt.Sprintf("LD_LIBRARY_PATH=%s:%s", filepath.Dir(path), os.ExpandEnv("$LD_LIBRARY_PATH")),
)
for _, nv := range opts.env {
env = append(env, fmt.Sprintf("%s=%s", nv[0], nv[1]))
}
// Hold any command line arguments to pass to the command.
cmdArgs := make([]string, 0, len(opts.args)*2)
if len(opts.args) > 0 {
for _, arg := range opts.args {
cmdArgs = append(cmdArgs, arg[0])
if arg[1] != "" {
cmdArgs = append(cmdArgs, arg[1])
}
}
}
// Run `env` to setup the environment.
args = append(args, "env")
args = append(args, env...)
// Pass the command and its arguments.
args = append(args, path)
args = append(args, cmdArgs...)
// Use sudo to run the command as the login user.
args = append([]string{"sudo"}, args...)
// If a timeout is set, prefix the command with "timeout".
if opts.timeout > 0 {
args = append([]string{"timeout", fmt.Sprintf("%ds", int(opts.timeout.Seconds()))}, args...)
}
cmd = exec.Command(args[0], args[1:]...) // #nosec G204
return
}
// run a command, passing its output to stdout and stderr.
func run(path string, opts eopts) (lastLogs string, err error) {
cmd, err := baserun(path, opts)
if err != nil {
return "", err
}
cmd.Stderr = os.Stderr
cmd.Stdout = os.Stdout
log.Info().Str("cmd", cmd.String()).Msg("running command")
if err := cmd.Start(); err != nil {
return "", fmt.Errorf("open path %q: %w", path, err)
}
// Reap the child process in a background goroutine. Orbit runs as root and
// only calls Start here (it monitors the desktop process separately, so this
// function must return after starting it). Without a corresponding Wait, every
// `sudo`/`timeout` child that exits becomes a zombie. When the desktop fails to
// start and Orbit respawns it in a loop, these zombies accumulate by the
// thousands. See https://github.com/fleetdm/fleet/issues/41796.
go func() {
if err := cmd.Wait(); err != nil {
log.Debug().Err(err).Msg("run cmd wait")
}
}()
return "", nil
}
// runWithOutput runs a command and return its output and exit code.
func runWithOutput(path string, opts eopts) (output []byte, exitCode int, err error) {
cmd, err := baserun(path, opts)
if err != nil {
return nil, -1, err
}
output, err = cmd.Output()
if err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
exitCode = exitErr.ExitCode()
return output, exitCode, fmt.Errorf("%q exited with code %d: %w", path, exitCode, err)
}
return output, -1, fmt.Errorf("%q error: %w", path, err)
}
return output, exitCode, nil
}
func getUserID(user string) (string, error) {
uid_, err := exec.Command("id", "-u", user).Output()
if err != nil {
return "", fmt.Errorf("failed to execute id command for %q: %w", user, err)
}
uid := strings.TrimSpace(string(uid_))
if uid == "" {
return "", errors.New("failed to get uid")
}
return uid, nil
}
func getDisplayVariableForSession(userID string, displaySessionType userpkg.GuiSessionType) string {
if displaySessionType == userpkg.GuiSessionTypeX11 {
x11Display, err := getUserX11Display(userID)
if err != nil {
log.Error().Err(err).Msg("failed to get X11 display, using default :0")
// TODO(lucas): Revisit when working on multi-user/multi-session support.
// Default to display ':0' if user display could not be found.
// This assumes there's only one desktop session and belongs to the
// user returned in `getLoginUID'.
return ":0"
}
return x11Display
}
waylandDisplay, err := getUserWaylandDisplay(userID)
if err != nil {
log.Error().Err(err).Msg("failed to get wayland display, using default wayland-0")
// TODO(lucas): Revisit when working on multi-user/multi-session support.
// Default to display 'wayland-0' if user display could not be found.
// This assumes there's only one desktop session and belongs to the
// user returned in `getLoginUID'.
return "wayland-0"
}
return waylandDisplay
}
func getConfigForCommand(user string, path string) (args []string, env []string, err error) {
// Get user ID
userID, err := getUserID(user)
if err != nil {
return nil, nil, fmt.Errorf("get user ID: %w", err)
}
log.Info().Str("user", user).Str("id", userID).Msg("attempting to get user session type and display")
// Get user's display session type.
userDisplaySession, err := userpkg.GetUserDisplaySessionType(userID)
if err != nil {
// Wayland is the default for most distributions,
// thus we assume wayland if we couldn't determine the session type.
log.Error().Err(err).Msg("assuming wayland session")
userDisplaySession = &userpkg.UserDisplaySession{
Type: userpkg.GuiSessionTypeWayland,
}
} else if userDisplaySession.Type == userpkg.GuiSessionTypeTty {
return nil, nil, fmt.Errorf("user %q (%s) is not running a GUI session", user, userID)
}
// Get user's "display" variable for the GUI session.
display := getDisplayVariableForSession(userID, userDisplaySession.Type)
log.Info().
Str("path", path).
Str("user", user).
Str("id", userID).
Str("display", display).
Str("session_type", userDisplaySession.Type.String()).
Msg("running sudo")
// On openSUSE Leap 16+ we drop -i (login shell). With -i, sudo runs the target
// user's shell as a login shell and passes the rest of the command via
// `bash --login -c`, which sources /etc/profile and /etc/profile.d/* and
// shell-escapes the inline command. On Leap 16 that environment indirection
// causes our `env KEY=val ... fleet-desktop` invocation to lose env vars, so
// fleet-desktop exits with "missing URL environment ..." and Orbit respawns it
// in a tight loop. -H sets HOME to the target user; sudo's default env_reset
// already sets USER/LOGNAME/SHELL.
//
// We keep -i on every other supported distribution to preserve the previously
// QA'd behavior.
if isOpenSUSELeap16Plus() {
args = []string{"-n", "-u", user, "-H"}
} else {
args = []string{"-n", "-i", "-u", user, "-H"}
}
env = make([]string, 0)
if userDisplaySession.Type == userpkg.GuiSessionTypeWayland {
env = append(env, "WAYLAND_DISPLAY="+display)
// For xdg-open to work on a Wayland session we still need to set the DISPLAY variable.
x11Display := ":" + strings.TrimPrefix(display, "wayland-")
env = append(env, "DISPLAY="+x11Display)
} else {
env = append(env, "DISPLAY="+display)
}
env = append(env,
// DBUS_SESSION_BUS_ADDRESS sets the location of the user login session bus.
// Required by the libayatana-appindicator3 library to display a tray icon
// on the desktop session.
//
// This is required for Ubuntu 18, and not required for Ubuntu 21/22
// (because it's already part of the user).
fmt.Sprintf("DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/%s/bus", userID),
)
return args, env, nil
}
// isOpenSUSELeap16Plus reports whether the host is running openSUSE Leap 16 or
// newer. We scope the no-login-shell sudo workaround to that distribution since
// it is the one observed to break under sudo -i; other distributions retain the
// previous (login-shell) launch path so we don't have to re-QA them.
func isOpenSUSELeap16Plus() bool {
data, err := os.ReadFile("/etc/os-release")
if err != nil {
return false
}
var id, versionID string
for line := range strings.SplitSeq(string(data), "\n") {
key, value, ok := strings.Cut(line, "=")
if !ok {
continue
}
// /etc/os-release values may be quoted.
value = strings.Trim(value, `"'`)
switch key {
case "ID":
id = value
case "VERSION_ID":
versionID = value
}
}
if id != "opensuse-leap" {
return false
}
// VERSION_ID is typically "16" or "16.0"; compare the major component.
major, _, _ := strings.Cut(versionID, ".")
n, err := strconv.Atoi(major)
if err != nil {
return false
}
return n >= 16
}
// getUserWaylandDisplay returns the value to set on WAYLAND_DISPLAY for the given user.
func getUserWaylandDisplay(uid string) (string, error) {
matches, err := filepath.Glob("/run/user/" + uid + "/wayland-*")
if err != nil {
return "", fmt.Errorf("list wayland socket files: %w", err)
}
sort.Slice(matches, func(i, j int) bool {
return matches[i] < matches[j]
})
for _, match := range matches {
if strings.HasSuffix(match, ".lock") {
continue
}
return filepath.Base(match), nil
}
return "", errors.New("wayland socket not found")
}
// getUserX11Display returns the value to set on DISPLAY for the given user.
// It scans /proc to find a process owned by the user that has DISPLAY set
// in its environment.
func getUserX11Display(userID string) (string, error) {
uid, err := strconv.ParseUint(userID, 10, 32)
if err != nil {
return "", fmt.Errorf("parse user ID %q: %w", userID, err)
}
entries, err := os.ReadDir("/proc")
if err != nil {
return "", fmt.Errorf("read /proc: %w", err)
}
for _, entry := range entries {
if !entry.IsDir() {
continue
}
// Skip non-PID directories.
if _, err := strconv.Atoi(entry.Name()); err != nil {
continue
}
// Check if the process belongs to our target user.
info, err := entry.Info()
if err != nil {
continue
}
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok || stat.Uid != uint32(uid) {
continue
}
// Try to read DISPLAY from this process's environment.
display, err := readEnvFromProc(entry.Name(), "DISPLAY")
if err != nil || display == "" {
continue
}
log.Debug().Msgf("found DISPLAY variable in %q", entry.Name())
return display, nil
}
return "", fmt.Errorf("DISPLAY not found in any process for user %s", userID)
}
// readEnvFromProc reads a specific environment variable from /proc/<pid>/environ.
func readEnvFromProc(pid string, envVar string) (string, error) {
return readEnvFromProcFile(fmt.Sprintf("/proc/%s/environ", pid), envVar)
}
// readEnvFromProcFile reads a specific environment variable from a /proc environ file.
// The file contains null-byte separated KEY=VALUE entries.
func readEnvFromProcFile(path string, envVar string) (string, error) {
data, err := os.ReadFile(path)
if err != nil {
return "", err
}
prefix := envVar + "="
for entry := range bytes.SplitSeq(data, []byte{0}) {
if s := string(entry); strings.HasPrefix(s, prefix) {
return s[len(prefix):], nil
}
}
return "", nil
}