<!-- Add the related story/sub-task/bug number, like Resolves #123, or remove if NA --> **Related issue:** Resolves #41796 Root cause: run() in orbit/pkg/execuser/execuser_linux.go called cmd.Start() but never cmd.Wait(). On Linux a child that exits stays a zombie until the parent reaps it, so every sudo … fleet-desktop invocation leaked a zombie. When Fleet Desktop fails to start, orbit respawns it in a loop (desktopRunner.Execute), so the zombies pile up. Fix: reap the child in a background goroutine after Start(). run() still returns immediately (orbit monitors the desktop process separately), so behavior is unchanged otherwise. # Checklist for submitter - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files) for more information. ## Testing - [x] QA'd all new/changed functionality manually #### Reproduction steps 1. Start TUF server and create .deb package ```sh SYSTEMS="linux-arm64" \ DEB_FLEET_URL=https://nicolasulmete.xyz \ # personal ngrok tunnel serving Fleet DEB_TUF_URL=http://nicotuf.ngrok.io \ # personal ngrok tunnel serving TUF GENERATE_DEB_ARM64=1 \ ENROLL_SECRET=<enroll-secret> \ FLEET_DESKTOP=1 \ TUF_PATH=$FLEET_REPO_DIR/test_tuf \ ./tools/tuf/test/main.sh ``` 2. Install .deb on a Ubuntu 25.10 ARM64 VM. 3. Push a failing fleet-desktop stub to trigger the respawn loop that produces zombie processes: ```sh export TUF_PATH=$FLEET_REPO_DIR/test_tuf export FLEET_TARGETS_PASSPHRASE=p4ssphr4s3 FLEET_SNAPSHOT_PASSPHRASE=p4ssphr4s3 FLEET_TIMESTAMP_PASSPHRASE=p4ssphr4s3 source ./tools/tuf/test/load_orbit_version_vars.sh STUB=$(mktemp -d) mkdir -p "$STUB/fleet-desktop" printf '#!/bin/sh\n[ "$1" = "--help" ] && exit 0\nexit 1\n' > "$STUB/fleet-desktop/fleet-desktop" chmod +x "$STUB/fleet-desktop/fleet-desktop" tar czf "$STUB/desktop.tar.gz" -C "$STUB" fleet-desktop ./build/fleetctl updates add --path "$TUF_PATH" --target "$STUB/desktop.tar.gz" --platform linux-arm64 --name desktop --version "$ORBIT_VERSION" -t "$ORBIT_MAJOR.$ORBIT_MINOR" -t "$ORBIT_MAJOR" -t stable rm -rf "$STUB" ``` 4. Watch zombie processes on the VM: ```sh watch -n5 'ORB=$(pgrep -x orbit); \ echo "zombie sudo children: $(ps -eo ppid,stat,comm | awk -v o="$ORB" "\$1==o && \$2 ~ /Z/ && \$3==\"sudo\"" | wc -l)"; \ echo "total sudo lines: $(ps aux | grep -c "[s]udo")"' ``` https://github.com/user-attachments/assets/dfa4d766-3c8e-4eb0-9104-7219c421538f #### Test fix 1. Switch to this branch, rebuild and push the new orbit binary (note that we still have the stub fleet-desktop that always exits). ```sh make build export TUF_PATH=$FLEET_REPO_DIR/test_tuf bash $FLEET_REPO_DIR/push-orbit-arm64.sh ``` Contents of `push-orbit-arm64.sh` is: ```sh #!/bin/bash set -e source ./tools/tuf/test/load_orbit_version_vars.sh GOOS=linux GOARCH=arm64 go build \ -o orbit-linux-arm64 \ -ldflags="-s -w -X github.com/fleetdm/fleet/v4/orbit/pkg/build.Version=$ORBIT_VERSION -X github.com/fleetdm/fleet/v4/orbit/pkg/build.Commit=$ORBIT_COMMIT" \ ./orbit/cmd/orbit ./tools/tuf/test/push_target.sh linux-arm64 orbit orbit-linux-arm64 "$ORBIT_VERSION" ``` 2. Verify on the VM that the update is detected and applied. Also, run step 4's cmd from the reproduction steps and see the zombies being 0, even though fleet-desktop is still continuously exiting. https://github.com/user-attachments/assets/d0f96d85-5e94-4878-b413-67031add3c1e ## fleetd/orbit/Fleet Desktop - [x] Verified compatibility with the latest released version of Fleet (see [Must rule](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/workflows/fleetd-development-and-release-strategy.md)) - [x] If the change applies to only one platform, confirmed that `runtime.GOOS` is used as needed to isolate changes - [x] Verified that fleetd runs on macOS, Linux and Windows - [x] Verified auto-update works from the released version of component to the new version (see [tools/tuf/test](../tools/tuf/test/README.md)) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **Bug Fixes** * Fixed an issue on Linux where repeated Fleet Desktop startup failures could result in thousands of orphaned processes accumulating in the background, degrading system performance. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
348 lines
11 KiB
Go
348 lines
11 KiB
Go
package execuser
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
|
|
userpkg "github.com/fleetdm/fleet/v4/orbit/pkg/user"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
// base command to setup an exec.Cmd using `runuser`
|
|
func baserun(path string, opts eopts) (cmd *exec.Cmd, err error) {
|
|
if opts.user == "" {
|
|
return nil, errors.New("missing user")
|
|
}
|
|
|
|
args, env, err := getConfigForCommand(opts.user, path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("get args: %w", err)
|
|
}
|
|
|
|
env = append(env,
|
|
// Append the packaged libayatana-appindicator3 libraries path to LD_LIBRARY_PATH.
|
|
//
|
|
// Fleet Desktop doesn't use libayatana-appindicator3 since 1.18.3, but we need to
|
|
// keep this to support older versions of Fleet Desktop.
|
|
fmt.Sprintf("LD_LIBRARY_PATH=%s:%s", filepath.Dir(path), os.ExpandEnv("$LD_LIBRARY_PATH")),
|
|
)
|
|
|
|
for _, nv := range opts.env {
|
|
env = append(env, fmt.Sprintf("%s=%s", nv[0], nv[1]))
|
|
}
|
|
|
|
// Hold any command line arguments to pass to the command.
|
|
cmdArgs := make([]string, 0, len(opts.args)*2)
|
|
if len(opts.args) > 0 {
|
|
for _, arg := range opts.args {
|
|
cmdArgs = append(cmdArgs, arg[0])
|
|
if arg[1] != "" {
|
|
cmdArgs = append(cmdArgs, arg[1])
|
|
}
|
|
}
|
|
}
|
|
|
|
// Run `env` to setup the environment.
|
|
args = append(args, "env")
|
|
args = append(args, env...)
|
|
// Pass the command and its arguments.
|
|
args = append(args, path)
|
|
args = append(args, cmdArgs...)
|
|
|
|
// Use sudo to run the command as the login user.
|
|
args = append([]string{"sudo"}, args...)
|
|
|
|
// If a timeout is set, prefix the command with "timeout".
|
|
if opts.timeout > 0 {
|
|
args = append([]string{"timeout", fmt.Sprintf("%ds", int(opts.timeout.Seconds()))}, args...)
|
|
}
|
|
|
|
cmd = exec.Command(args[0], args[1:]...) // #nosec G204
|
|
return
|
|
}
|
|
|
|
// run a command, passing its output to stdout and stderr.
|
|
func run(path string, opts eopts) (lastLogs string, err error) {
|
|
cmd, err := baserun(path, opts)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
cmd.Stderr = os.Stderr
|
|
cmd.Stdout = os.Stdout
|
|
log.Info().Str("cmd", cmd.String()).Msg("running command")
|
|
|
|
if err := cmd.Start(); err != nil {
|
|
return "", fmt.Errorf("open path %q: %w", path, err)
|
|
}
|
|
// Reap the child process in a background goroutine. Orbit runs as root and
|
|
// only calls Start here (it monitors the desktop process separately, so this
|
|
// function must return after starting it). Without a corresponding Wait, every
|
|
// `sudo`/`timeout` child that exits becomes a zombie. When the desktop fails to
|
|
// start and Orbit respawns it in a loop, these zombies accumulate by the
|
|
// thousands. See https://github.com/fleetdm/fleet/issues/41796.
|
|
go func() {
|
|
if err := cmd.Wait(); err != nil {
|
|
log.Debug().Err(err).Msg("run cmd wait")
|
|
}
|
|
}()
|
|
return "", nil
|
|
}
|
|
|
|
// runWithOutput runs a command and return its output and exit code.
|
|
func runWithOutput(path string, opts eopts) (output []byte, exitCode int, err error) {
|
|
cmd, err := baserun(path, opts)
|
|
if err != nil {
|
|
return nil, -1, err
|
|
}
|
|
|
|
output, err = cmd.Output()
|
|
if err != nil {
|
|
if exitErr, ok := err.(*exec.ExitError); ok {
|
|
exitCode = exitErr.ExitCode()
|
|
return output, exitCode, fmt.Errorf("%q exited with code %d: %w", path, exitCode, err)
|
|
}
|
|
return output, -1, fmt.Errorf("%q error: %w", path, err)
|
|
}
|
|
|
|
return output, exitCode, nil
|
|
}
|
|
|
|
func getUserID(user string) (string, error) {
|
|
uid_, err := exec.Command("id", "-u", user).Output()
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to execute id command for %q: %w", user, err)
|
|
}
|
|
uid := strings.TrimSpace(string(uid_))
|
|
if uid == "" {
|
|
return "", errors.New("failed to get uid")
|
|
}
|
|
return uid, nil
|
|
}
|
|
|
|
func getDisplayVariableForSession(userID string, displaySessionType userpkg.GuiSessionType) string {
|
|
if displaySessionType == userpkg.GuiSessionTypeX11 {
|
|
x11Display, err := getUserX11Display(userID)
|
|
if err != nil {
|
|
log.Error().Err(err).Msg("failed to get X11 display, using default :0")
|
|
// TODO(lucas): Revisit when working on multi-user/multi-session support.
|
|
// Default to display ':0' if user display could not be found.
|
|
// This assumes there's only one desktop session and belongs to the
|
|
// user returned in `getLoginUID'.
|
|
return ":0"
|
|
}
|
|
return x11Display
|
|
}
|
|
|
|
waylandDisplay, err := getUserWaylandDisplay(userID)
|
|
if err != nil {
|
|
log.Error().Err(err).Msg("failed to get wayland display, using default wayland-0")
|
|
// TODO(lucas): Revisit when working on multi-user/multi-session support.
|
|
// Default to display 'wayland-0' if user display could not be found.
|
|
// This assumes there's only one desktop session and belongs to the
|
|
// user returned in `getLoginUID'.
|
|
return "wayland-0"
|
|
}
|
|
return waylandDisplay
|
|
}
|
|
|
|
func getConfigForCommand(user string, path string) (args []string, env []string, err error) {
|
|
// Get user ID
|
|
userID, err := getUserID(user)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("get user ID: %w", err)
|
|
}
|
|
log.Info().Str("user", user).Str("id", userID).Msg("attempting to get user session type and display")
|
|
|
|
// Get user's display session type.
|
|
userDisplaySession, err := userpkg.GetUserDisplaySessionType(userID)
|
|
if err != nil {
|
|
// Wayland is the default for most distributions,
|
|
// thus we assume wayland if we couldn't determine the session type.
|
|
log.Error().Err(err).Msg("assuming wayland session")
|
|
userDisplaySession = &userpkg.UserDisplaySession{
|
|
Type: userpkg.GuiSessionTypeWayland,
|
|
}
|
|
} else if userDisplaySession.Type == userpkg.GuiSessionTypeTty {
|
|
return nil, nil, fmt.Errorf("user %q (%s) is not running a GUI session", user, userID)
|
|
}
|
|
|
|
// Get user's "display" variable for the GUI session.
|
|
display := getDisplayVariableForSession(userID, userDisplaySession.Type)
|
|
|
|
log.Info().
|
|
Str("path", path).
|
|
Str("user", user).
|
|
Str("id", userID).
|
|
Str("display", display).
|
|
Str("session_type", userDisplaySession.Type.String()).
|
|
Msg("running sudo")
|
|
|
|
// On openSUSE Leap 16+ we drop -i (login shell). With -i, sudo runs the target
|
|
// user's shell as a login shell and passes the rest of the command via
|
|
// `bash --login -c`, which sources /etc/profile and /etc/profile.d/* and
|
|
// shell-escapes the inline command. On Leap 16 that environment indirection
|
|
// causes our `env KEY=val ... fleet-desktop` invocation to lose env vars, so
|
|
// fleet-desktop exits with "missing URL environment ..." and Orbit respawns it
|
|
// in a tight loop. -H sets HOME to the target user; sudo's default env_reset
|
|
// already sets USER/LOGNAME/SHELL.
|
|
//
|
|
// We keep -i on every other supported distribution to preserve the previously
|
|
// QA'd behavior.
|
|
if isOpenSUSELeap16Plus() {
|
|
args = []string{"-n", "-u", user, "-H"}
|
|
} else {
|
|
args = []string{"-n", "-i", "-u", user, "-H"}
|
|
}
|
|
env = make([]string, 0)
|
|
|
|
if userDisplaySession.Type == userpkg.GuiSessionTypeWayland {
|
|
env = append(env, "WAYLAND_DISPLAY="+display)
|
|
// For xdg-open to work on a Wayland session we still need to set the DISPLAY variable.
|
|
x11Display := ":" + strings.TrimPrefix(display, "wayland-")
|
|
env = append(env, "DISPLAY="+x11Display)
|
|
} else {
|
|
env = append(env, "DISPLAY="+display)
|
|
}
|
|
|
|
env = append(env,
|
|
// DBUS_SESSION_BUS_ADDRESS sets the location of the user login session bus.
|
|
// Required by the libayatana-appindicator3 library to display a tray icon
|
|
// on the desktop session.
|
|
//
|
|
// This is required for Ubuntu 18, and not required for Ubuntu 21/22
|
|
// (because it's already part of the user).
|
|
fmt.Sprintf("DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/%s/bus", userID),
|
|
)
|
|
|
|
return args, env, nil
|
|
}
|
|
|
|
// isOpenSUSELeap16Plus reports whether the host is running openSUSE Leap 16 or
|
|
// newer. We scope the no-login-shell sudo workaround to that distribution since
|
|
// it is the one observed to break under sudo -i; other distributions retain the
|
|
// previous (login-shell) launch path so we don't have to re-QA them.
|
|
func isOpenSUSELeap16Plus() bool {
|
|
data, err := os.ReadFile("/etc/os-release")
|
|
if err != nil {
|
|
return false
|
|
}
|
|
var id, versionID string
|
|
for line := range strings.SplitSeq(string(data), "\n") {
|
|
key, value, ok := strings.Cut(line, "=")
|
|
if !ok {
|
|
continue
|
|
}
|
|
// /etc/os-release values may be quoted.
|
|
value = strings.Trim(value, `"'`)
|
|
switch key {
|
|
case "ID":
|
|
id = value
|
|
case "VERSION_ID":
|
|
versionID = value
|
|
}
|
|
}
|
|
if id != "opensuse-leap" {
|
|
return false
|
|
}
|
|
// VERSION_ID is typically "16" or "16.0"; compare the major component.
|
|
major, _, _ := strings.Cut(versionID, ".")
|
|
n, err := strconv.Atoi(major)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
return n >= 16
|
|
}
|
|
|
|
// getUserWaylandDisplay returns the value to set on WAYLAND_DISPLAY for the given user.
|
|
func getUserWaylandDisplay(uid string) (string, error) {
|
|
matches, err := filepath.Glob("/run/user/" + uid + "/wayland-*")
|
|
if err != nil {
|
|
return "", fmt.Errorf("list wayland socket files: %w", err)
|
|
}
|
|
sort.Slice(matches, func(i, j int) bool {
|
|
return matches[i] < matches[j]
|
|
})
|
|
for _, match := range matches {
|
|
if strings.HasSuffix(match, ".lock") {
|
|
continue
|
|
}
|
|
return filepath.Base(match), nil
|
|
}
|
|
return "", errors.New("wayland socket not found")
|
|
}
|
|
|
|
// getUserX11Display returns the value to set on DISPLAY for the given user.
|
|
// It scans /proc to find a process owned by the user that has DISPLAY set
|
|
// in its environment.
|
|
func getUserX11Display(userID string) (string, error) {
|
|
uid, err := strconv.ParseUint(userID, 10, 32)
|
|
if err != nil {
|
|
return "", fmt.Errorf("parse user ID %q: %w", userID, err)
|
|
}
|
|
|
|
entries, err := os.ReadDir("/proc")
|
|
if err != nil {
|
|
return "", fmt.Errorf("read /proc: %w", err)
|
|
}
|
|
|
|
for _, entry := range entries {
|
|
if !entry.IsDir() {
|
|
continue
|
|
}
|
|
// Skip non-PID directories.
|
|
if _, err := strconv.Atoi(entry.Name()); err != nil {
|
|
continue
|
|
}
|
|
// Check if the process belongs to our target user.
|
|
info, err := entry.Info()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
stat, ok := info.Sys().(*syscall.Stat_t)
|
|
if !ok || stat.Uid != uint32(uid) {
|
|
continue
|
|
}
|
|
|
|
// Try to read DISPLAY from this process's environment.
|
|
display, err := readEnvFromProc(entry.Name(), "DISPLAY")
|
|
if err != nil || display == "" {
|
|
continue
|
|
}
|
|
|
|
log.Debug().Msgf("found DISPLAY variable in %q", entry.Name())
|
|
return display, nil
|
|
}
|
|
|
|
return "", fmt.Errorf("DISPLAY not found in any process for user %s", userID)
|
|
}
|
|
|
|
// readEnvFromProc reads a specific environment variable from /proc/<pid>/environ.
|
|
func readEnvFromProc(pid string, envVar string) (string, error) {
|
|
return readEnvFromProcFile(fmt.Sprintf("/proc/%s/environ", pid), envVar)
|
|
}
|
|
|
|
// readEnvFromProcFile reads a specific environment variable from a /proc environ file.
|
|
// The file contains null-byte separated KEY=VALUE entries.
|
|
func readEnvFromProcFile(path string, envVar string) (string, error) {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
prefix := envVar + "="
|
|
for entry := range bytes.SplitSeq(data, []byte{0}) {
|
|
if s := string(entry); strings.HasPrefix(s, prefix) {
|
|
return s[len(prefix):], nil
|
|
}
|
|
}
|
|
return "", nil
|
|
}
|