When building Linux and macOS fleetd packages, removed duplicate copies of osqueryd and fleet-desktop (#32697)

Fixes #32280 

- Removed osqueryd.tar.gz from macOS package and desktop.tar.gz from
macOS and Linux packages and replaced them with .sha512 hash caches.

# Checklist for submitter

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

## Testing

- [x] QA'd all new/changed functionality manually

## fleetd/orbit/Fleet Desktop

- [x] Verified compatibility with the latest released version of Fleet
(see [Must
rule](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/workflows/fleetd-development-and-release-strategy.md))
- [x] Verified that fleetd runs on macOS, Linux and Windows
- [x] Verified auto-update works from the released version of component
to the new version (see [tools/tuf/test](../tools/tuf/test/README.md))

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **Bug Fixes**
* Eliminated duplicate osqueryd and Fleet Desktop binaries in Linux and
macOS packages, preventing duplicate entries in .deb/.pkg and ensuring
cleaner installs.

* **Chores**
* Added packaging cleanup to remove leftover tar.gz artifacts, reducing
package size and avoiding accidental inclusion in builds.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

changes/32280-duplicate-desktop-osqueryd

@@ -0,0 +1 @@
+* Linux/macOS packaging: removed duplicate tar.gz copies of osqueryd and Fleet Desktop from built packages (DEB/RPM/PKG).

orbit/changes/32280-duplicate-desktop-osqueryd

@@ -0,0 +1 @@
+* Since new macOS/Linux packages built with `fleetctl 4.75.0` or higher do not have embedded osqueryd.app.tar.gz and desktop.tar.gz, orbit can now use osqueryd.app.tar.gz.sha512 and desktop.tar.gz.sha512/desktop.app.tar.gz.sha512 hash caches to check if an update is needed.

orbit/pkg/packaging/packaging.go

@@ -5,11 +5,15 @@
 package packaging
 
 import (
+	"crypto/sha512"
 	_ "embed"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/fleetdm/fleet/v4/orbit/pkg/constant"
@@ -211,6 +215,14 @@ func InitializeUpdates(updateOpt update.Options) (*UpdatesData, error) {
 		return nil, fmt.Errorf("failed to get %s version: %w", constant.OsqueryTUFTargetName, err)
 	}
 
+	// Save hash and remove osqueryd tar.gz to prevent it from being included in the package
+	// (on macOS, osqueryd comes as osqueryd.app.tar.gz)
+	if strings.HasSuffix(osquerydLocalTarget.Path, ".tar.gz") {
+		if err := saveHashAndRemoveTarGz(osquerydLocalTarget.Path); err != nil {
+			log.Error().Err(err).Str("path", osquerydLocalTarget.Path).Msg("failed to save hash and remove osqueryd tar.gz")
+		}
+	}
+
 	orbitLocalTarget, err := updater.Get(constant.OrbitTUFTargetName)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get %s: %w", constant.OrbitTUFTargetName, err)
@@ -242,6 +254,14 @@ func InitializeUpdates(updateOpt update.Options) (*UpdatesData, error) {
 		if err := json.Unmarshal(*desktopMeta.Custom, &desktopCustom); err != nil {
 			return nil, fmt.Errorf("failed to get %s version: %w", constant.DesktopTUFTargetName, err)
 		}
+
+		// Save hash and remove the tar.gz file to prevent it from being included in the package
+		// (fixes duplicate fleet-desktop in .deb and .pkg packages)
+		if strings.HasSuffix(desktopLocalTarget.Path, ".tar.gz") {
+			if err := saveHashAndRemoveTarGz(desktopLocalTarget.Path); err != nil {
+				log.Error().Err(err).Str("path", desktopLocalTarget.Path).Msg("failed to save hash and remove desktop tar.gz")
+			}
+		}
 	}
 
 	// Copy the new metadata file to the old location (pre-migration) to
@@ -267,6 +287,45 @@ func InitializeUpdates(updateOpt update.Options) (*UpdatesData, error) {
 	}, nil
 }
 
+// saveHashAndRemoveTarGz calculates the SHA512 hash of a tar.gz file,
+// saves it to a .sha512 file, then removes the tar.gz.
+// This allows orbit to verify integrity on first run without keeping duplicate tar.gz files.
+func saveHashAndRemoveTarGz(tarGzPath string) error {
+	// Open the tar.gz file
+	f, err := os.Open(tarGzPath)
+	if err != nil {
+		return fmt.Errorf("open tar.gz for hashing: %w", err)
+	}
+	defer f.Close()
+
+	// Calculate SHA512 (currently the only hash algorithm used by Fleet TUF)
+	sha512Hash := sha512.New()
+	if _, err := io.Copy(sha512Hash, f); err != nil {
+		return fmt.Errorf("hash tar.gz: %w", err)
+	}
+
+	// Save SHA512 hash
+	sha512Path := tarGzPath + ".sha512"
+	sha512Hex := hex.EncodeToString(sha512Hash.Sum(nil))
+	if err := os.WriteFile(sha512Path, []byte(sha512Hex), constant.DefaultFileMode); err != nil {
+		return fmt.Errorf("write sha512 file: %w", err)
+	}
+
+	// Remove the tar.gz file
+	if err := os.Remove(tarGzPath); err != nil {
+		// Clean up hash file if we fail to remove tar.gz
+		_ = os.Remove(sha512Path)
+		return fmt.Errorf("remove tar.gz: %w", err)
+	}
+
+	log.Debug().
+		Str("path", tarGzPath).
+		Str("sha512", sha512Hex).
+		Msg("saved hash and removed tar.gz")
+
+	return nil
+}
+
 // writeSecret writes the orbit enroll secret to the designated file.
 //
 // This implementation is very similar to the one in orbit/cmd/orbit but

orbit/pkg/update/hash.go

@@ -4,11 +4,14 @@ import (
 	"bytes"
 	"crypto/sha256"
 	"crypto/sha512"
+	"encoding/hex"
 	"fmt"
 	"hash"
 	"io"
 	"os"
+	"strings"
 
+	"github.com/rs/zerolog/log"
 	"github.com/theupdateframework/go-tuf/data"
 )
 
@@ -32,6 +35,14 @@ func fileHashes(meta *data.TargetFileMeta, localPath string) (metaHash []byte, l
 
 	f, err := os.Open(localPath)
 	if err != nil {
+		// If tar.gz doesn't exist but a hash file does, use the cached hash file
+		if os.IsNotExist(err) && strings.HasSuffix(localPath, ".tar.gz") {
+			cachedHash, err := readCachedHash(localPath, meta)
+			if err == nil {
+				return metaHash, cachedHash, nil
+			}
+			log.Info().Err(err).Msg("failed to read cached hash file")
+		}
 		return nil, nil, fmt.Errorf("open file for hash: %w", err)
 	}
 	defer f.Close()
@@ -61,3 +72,22 @@ func selectHashFunction(meta *data.TargetFileMeta) (hash.Hash, []byte, error) {
 
 	return nil, nil, fmt.Errorf("no matching hash function found: %v", meta.HashAlgorithms())
 }
+
+// readCachedHash reads a cached hash from a .sha512 file
+// created during packaging when the tar.gz was removed to save space.
+func readCachedHash(tarGzPath string, meta *data.TargetFileMeta) ([]byte, error) {
+	// Check if TUF metadata has SHA512 (currently the only hash file used)
+	for hashName := range meta.Hashes {
+		if hashName == "sha512" {
+			hashPath := tarGzPath + ".sha512"
+			var hashHex []byte
+			var err error
+			if hashHex, err = os.ReadFile(hashPath); err != nil {
+				return nil, err
+			}
+			return hex.DecodeString(strings.TrimSpace(string(hashHex)))
+		}
+	}
+
+	return nil, fmt.Errorf("no cached hash file found for %s", tarGzPath)
+}

orbit/pkg/update/update.go

@@ -463,6 +463,8 @@ func (u *Updater) get(target string) (*LocalTarget, error) {
 				return nil, fmt.Errorf("download %q: %w", repoPath, err)
 			}
 			if strings.HasSuffix(localTarget.Path, ".tar.gz") {
+				// Remove cached hash files since we have a real tar.gz now
+				removeCachedHashes(localTarget.Path)
 				if err := os.RemoveAll(localTarget.DirPath); err != nil {
 					return nil, fmt.Errorf("failed to remove old extracted dir: %q: %w", localTarget.DirPath, err)
 				}
@@ -476,9 +478,39 @@ func (u *Updater) get(target string) (*LocalTarget, error) {
 			log.Debug().Str("path", localTarget.Path).Str("target", target).Msg("found expected target locally")
 		}
 	case errors.Is(err, os.ErrNotExist):
-		log.Debug().Err(err).Msg("stat file")
-		if err := u.download(target, repoPath, localTarget.Path, localTarget.Info.CustomCheckExec); err != nil {
-			return nil, fmt.Errorf("download %q: %w", repoPath, err)
+		// Check if we have a cached hash file for tar.gz files
+		if strings.HasSuffix(localTarget.Path, ".tar.gz") {
+			hashPath := localTarget.Path + ".sha512"
+			if _, hashErr := os.Stat(hashPath); hashErr == nil {
+				// We have a hash file, so check if it matches TUF metadata
+				meta, err := u.Lookup(target)
+				if err != nil {
+					return nil, err
+				}
+				if err := checkFileHash(meta, localTarget.Path); err != nil {
+					// Hash doesn't match or can't be verified, download the tar.gz
+					log.Debug().Str("info", err.Error()).Msg("hash mismatch or verification failed, downloading")
+					if err := u.download(target, repoPath, localTarget.Path, localTarget.Info.CustomCheckExec); err != nil {
+						return nil, fmt.Errorf("download %q: %w", repoPath, err)
+					}
+					removeCachedHashes(localTarget.Path)
+				} else {
+					// Hash matches! We can proceed without the tar.gz
+					log.Debug().Str("path", localTarget.Path).Msg("using cached hash, tar.gz not needed")
+				}
+			} else {
+				// No hash file either, need to download
+				log.Debug().Err(err).Msg("no tar.gz or hash file, downloading")
+				if err := u.download(target, repoPath, localTarget.Path, localTarget.Info.CustomCheckExec); err != nil {
+					return nil, fmt.Errorf("download %q: %w", repoPath, err)
+				}
+			}
+		} else {
+			// Not a tar.gz, just download it
+			log.Debug().Err(err).Msg("stat file")
+			if err := u.download(target, repoPath, localTarget.Path, localTarget.Info.CustomCheckExec); err != nil {
+				return nil, fmt.Errorf("download %q: %w", repoPath, err)
+			}
 		}
 		if strings.HasSuffix(localTarget.Path, ".pkg") && runtime.GOOS == "darwin" {
 			if err := installPKG(localTarget.Path); err != nil {
@@ -493,8 +525,16 @@ func (u *Updater) get(target string) (*LocalTarget, error) {
 		s, err := os.Stat(localTarget.ExecPath)
 		switch {
 		case err == nil:
-			// OK
+			// OK - executable exists
 		case errors.Is(err, os.ErrNotExist):
+			// Check if tar.gz exists before trying to extract
+			if _, tarErr := os.Stat(localTarget.Path); tarErr != nil {
+				// No tar.gz to extract from.
+				// The executable should already be in the initial package, and this error should never happen under normal circumstances.
+				// Delete the .sha512 file so next run will download the tar.gz
+				removeCachedHashes(localTarget.Path)
+				return nil, fmt.Errorf("executable not found and no tar.gz to extract: %q", localTarget.ExecPath)
+			}
 			if err := extractTarGz(localTarget.Path); err != nil {
 				return nil, fmt.Errorf("extract %q: %w", localTarget.Path, err)
 			}
@@ -741,6 +781,13 @@ func extractTarGz(path string) error {
 	}
 }
 
+// removeCachedHashes removes the .sha512 file that was created
+// during packaging to cache the hash when the tar.gz was removed.
+func removeCachedHashes(tarGzPath string) {
+	// Remove hash file, ignore errors (file may not exist)
+	_ = os.Remove(tarGzPath + ".sha512")
+}
+
 func installPKG(path string) error {
 	cmd := exec.Command("installer", "-pkg", path, "-target", "/")
 	if out, err := cmd.CombinedOutput(); err != nil {