From 6e227b6eb501da382011b0aae11cbee72a07678f Mon Sep 17 00:00:00 2001 From: Dan Fuhry Date: Mon, 28 Jul 2025 09:55:44 -0400 Subject: [PATCH] [orbit/packaging] delay restart for in-band pkg upgrade on Linux (#31286) Admins _should_ be upgrading orbit/osqueryd/fleet-desktop using TUF, but there's no official path for pushing updates to the orbit environment file (`/etc/default/orbit`). Knowing that this file is installed by fleet-osquery, I naively pushed a new fleet-osquery package to a user's machine, thinking that would be fine installing over the existing package. Instead it actually broke orbit entirely on the host, rendering it unreachable until the user manually reinstalled the package. This is because the pre-removal script unconditionally stops orbit even if it's being upgraded, and when orbit is stopped it terminates any processes underneath it, including the ongoing package installation. To workaround this, we replace the simple `systemctl restart orbit.service` with a check for the `INSTALLER_PATH` environment variable that orbit sets during software installations. If the variable is present, `systemd-run` is used to schedule the service restart 60 seconds in the future, which is assumed to be more than enough time for the package manager to finish and exit. Unfortunately, this bugfix cannot be made retroactive, because the broken version of the prerm script is called before the new package starts to be unpacked/installed. Although there are other ways of doing anything an administrator might be trying to accomplish by pushing a new fleet-osquery package, bricking an endpoint simply by pushing this package through the Software page seems like a pretty massive footgun that is easy to protect against. ## Summary by CodeRabbit * **New Features** * Added support for safe in-band upgrades of DEB and RPM packages generated by the `fleetctl package` command via the Software page, after an initial manual update. * **Documentation** * Added a note clarifying the new upgrade process and the need for a one-time manual update before using in-band upgrades. * **Bug Fixes** * Improved upgrade scripts to prevent the Orbit service from stopping unexpectedly during package upgrades, ensuring smoother and safer updates. Signed-off-by: Dan Fuhry --- orbit/changes/31286-package-upgrade-fix | 3 ++ orbit/pkg/packaging/linux_shared.go | 52 ++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 orbit/changes/31286-package-upgrade-fix diff --git a/orbit/changes/31286-package-upgrade-fix b/orbit/changes/31286-package-upgrade-fix new file mode 100644 index 0000000000..0ef9cb64b3 --- /dev/null +++ b/orbit/changes/31286-package-upgrade-fix @@ -0,0 +1,3 @@ +- DEB and RPM packages generated by `fleetctl package` will now be safe to upgrade in-band through the Software page. + - Note that the package will need to be updated out-of-band once, because the pre-removal script from previously-generated packages is called upon an upgrade. The old pre-removal script stopped Orbit unconditionally. + - In other words, fleet-osquery can safely be updated through the Software page only _after_ a new package generated with this version of fleetctl has been installed through other means. diff --git a/orbit/pkg/packaging/linux_shared.go b/orbit/pkg/packaging/linux_shared.go index aea847ceef..606c6494b5 100644 --- a/orbit/pkg/packaging/linux_shared.go +++ b/orbit/pkg/packaging/linux_shared.go @@ -19,6 +19,25 @@ import ( "github.com/rs/zerolog/log" ) +// Reusable snippet to conditionally wait to restart orbit upon an in-band upgrade +// (orbit installing an update to itself). Without this, the maintainer script will +// terminate orbit along with the package installation, aborting the install and +// potentially leaving the host in an unreachable state. +const postInstallSafeRestart = ` + if test -z "${INSTALLER_PATH:-}"; then + systemctl restart orbit.service 2>&1 + else + echo "Detected in-band upgrade (orbit upgrading orbit). Delaying service" + echo "restart to prevent orbit from being stopped mid-script." + if command -v systemd-run >/dev/null 2>&1; then + systemd-run --on-active=60 --working-directory=/ systemctl restart --no-block orbit.service + else + echo "...nevermind, systemd-run not available, exiting postinst" + echo "without restarting." + fi + fi +` + func buildNFPM(opt Options, pkger nfpm.Packager) (string, error) { // Initialize directories tmpDir, err := initializeTempDir() @@ -367,7 +386,7 @@ set -e if command -v systemctl >/dev/null 2>&1; then systemctl daemon-reload >/dev/null 2>&1 {{ if .StartService -}} - systemctl restart orbit.service 2>&1 + ` + postInstallSafeRestart + ` systemctl enable orbit.service 2>&1 {{- end}} fi @@ -391,14 +410,37 @@ func writePreRemove(opt Options, path string) error { // or has been manually disabled already. Otherwise, // uninstallation fails. // + // Upgrades require special considerations. + // + // On Debian systems, the old package is removed BEFORE the + // new package is installed. The old package's prerm script is + // called with the first argument set to "upgrade" if the + // package is being upgraded. In this case, we do not stop or + // disable the service; it will be restarted by the post-install + // script. If called with "remove" or "deconfigure", we should + // stop and disable any running orbit service. + // https://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#details-of-unpack-phase-of-installation-or-upgrade + // + // On RPM systems, the old package is removed AFTER the new + // package is installed. The preun script is called with the + // argument "0" upon uninstall and "1" upon upgrade. + // https://docs.fedoraproject.org/en-US/packaging-guidelines/Scriptlets/#_syntax + // https://docs.fedoraproject.org/en-US/packaging-guidelines/Scriptlets/#ordering + // // "pkill fleet-desktop" is required because the application // runs as user (separate from sudo command that launched it), // so on some systems it's not killed properly. if err := os.WriteFile(path, []byte(`#!/bin/sh -systemctl stop orbit.service || true -systemctl disable orbit.service || true -pkill fleet-desktop || true +case "${1:-}" in + 1|remove|deconfigure) + systemctl disable --now orbit.service || true + pkill fleet-desktop || true + ;; + 0|upgrade) + ;; +esac + `), constant.DefaultFileMode); err != nil { return fmt.Errorf("write file: %w", err) } @@ -438,7 +480,7 @@ if ! systemctl is-enabled orbit >/dev/null 2>&1; then if command -v systemctl >/dev/null 2>&1; then systemctl daemon-reload >/dev/null 2>&1 {{ if .StartService -}} - systemctl restart orbit.service 2>&1 + ` + postInstallSafeRestart + ` systemctl enable orbit.service 2>&1 {{- end}} fi