diff --git a/changes/32280-duplicate-desktop-osqueryd b/changes/32280-duplicate-desktop-osqueryd new file mode 100644 index 0000000000..fe5993542e --- /dev/null +++ b/changes/32280-duplicate-desktop-osqueryd @@ -0,0 +1 @@ +* Linux/macOS packaging: removed duplicate tar.gz copies of osqueryd and Fleet Desktop from built packages (DEB/RPM/PKG). diff --git a/orbit/changes/32280-duplicate-desktop-osqueryd b/orbit/changes/32280-duplicate-desktop-osqueryd new file mode 100644 index 0000000000..57968fc2c3 --- /dev/null +++ b/orbit/changes/32280-duplicate-desktop-osqueryd @@ -0,0 +1 @@ +* Since new macOS/Linux packages built with `fleetctl 4.75.0` or higher do not have embedded osqueryd.app.tar.gz and desktop.tar.gz, orbit can now use osqueryd.app.tar.gz.sha512 and desktop.tar.gz.sha512/desktop.app.tar.gz.sha512 hash caches to check if an update is needed. diff --git a/orbit/pkg/packaging/packaging.go b/orbit/pkg/packaging/packaging.go index 6e5a2644dc..cd81341f6d 100644 --- a/orbit/pkg/packaging/packaging.go +++ b/orbit/pkg/packaging/packaging.go @@ -5,11 +5,15 @@ package packaging import ( + "crypto/sha512" _ "embed" + "encoding/hex" "encoding/json" "fmt" + "io" "os" "path/filepath" + "strings" "time" "github.com/fleetdm/fleet/v4/orbit/pkg/constant" @@ -211,6 +215,14 @@ func InitializeUpdates(updateOpt update.Options) (*UpdatesData, error) { return nil, fmt.Errorf("failed to get %s version: %w", constant.OsqueryTUFTargetName, err) } + // Save hash and remove osqueryd tar.gz to prevent it from being included in the package + // (on macOS, osqueryd comes as osqueryd.app.tar.gz) + if strings.HasSuffix(osquerydLocalTarget.Path, ".tar.gz") { + if err := saveHashAndRemoveTarGz(osquerydLocalTarget.Path); err != nil { + log.Error().Err(err).Str("path", osquerydLocalTarget.Path).Msg("failed to save hash and remove osqueryd tar.gz") + } + } + orbitLocalTarget, err := updater.Get(constant.OrbitTUFTargetName) if err != nil { return nil, fmt.Errorf("failed to get %s: %w", constant.OrbitTUFTargetName, err) @@ -242,6 +254,14 @@ func InitializeUpdates(updateOpt update.Options) (*UpdatesData, error) { if err := json.Unmarshal(*desktopMeta.Custom, &desktopCustom); err != nil { return nil, fmt.Errorf("failed to get %s version: %w", constant.DesktopTUFTargetName, err) } + + // Save hash and remove the tar.gz file to prevent it from being included in the package + // (fixes duplicate fleet-desktop in .deb and .pkg packages) + if strings.HasSuffix(desktopLocalTarget.Path, ".tar.gz") { + if err := saveHashAndRemoveTarGz(desktopLocalTarget.Path); err != nil { + log.Error().Err(err).Str("path", desktopLocalTarget.Path).Msg("failed to save hash and remove desktop tar.gz") + } + } } // Copy the new metadata file to the old location (pre-migration) to @@ -267,6 +287,45 @@ func InitializeUpdates(updateOpt update.Options) (*UpdatesData, error) { }, nil } +// saveHashAndRemoveTarGz calculates the SHA512 hash of a tar.gz file, +// saves it to a .sha512 file, then removes the tar.gz. +// This allows orbit to verify integrity on first run without keeping duplicate tar.gz files. +func saveHashAndRemoveTarGz(tarGzPath string) error { + // Open the tar.gz file + f, err := os.Open(tarGzPath) + if err != nil { + return fmt.Errorf("open tar.gz for hashing: %w", err) + } + defer f.Close() + + // Calculate SHA512 (currently the only hash algorithm used by Fleet TUF) + sha512Hash := sha512.New() + if _, err := io.Copy(sha512Hash, f); err != nil { + return fmt.Errorf("hash tar.gz: %w", err) + } + + // Save SHA512 hash + sha512Path := tarGzPath + ".sha512" + sha512Hex := hex.EncodeToString(sha512Hash.Sum(nil)) + if err := os.WriteFile(sha512Path, []byte(sha512Hex), constant.DefaultFileMode); err != nil { + return fmt.Errorf("write sha512 file: %w", err) + } + + // Remove the tar.gz file + if err := os.Remove(tarGzPath); err != nil { + // Clean up hash file if we fail to remove tar.gz + _ = os.Remove(sha512Path) + return fmt.Errorf("remove tar.gz: %w", err) + } + + log.Debug(). + Str("path", tarGzPath). + Str("sha512", sha512Hex). + Msg("saved hash and removed tar.gz") + + return nil +} + // writeSecret writes the orbit enroll secret to the designated file. // // This implementation is very similar to the one in orbit/cmd/orbit but diff --git a/orbit/pkg/update/hash.go b/orbit/pkg/update/hash.go index f721e456ec..0865ac96e6 100644 --- a/orbit/pkg/update/hash.go +++ b/orbit/pkg/update/hash.go @@ -4,11 +4,14 @@ import ( "bytes" "crypto/sha256" "crypto/sha512" + "encoding/hex" "fmt" "hash" "io" "os" + "strings" + "github.com/rs/zerolog/log" "github.com/theupdateframework/go-tuf/data" ) @@ -32,6 +35,14 @@ func fileHashes(meta *data.TargetFileMeta, localPath string) (metaHash []byte, l f, err := os.Open(localPath) if err != nil { + // If tar.gz doesn't exist but a hash file does, use the cached hash file + if os.IsNotExist(err) && strings.HasSuffix(localPath, ".tar.gz") { + cachedHash, err := readCachedHash(localPath, meta) + if err == nil { + return metaHash, cachedHash, nil + } + log.Info().Err(err).Msg("failed to read cached hash file") + } return nil, nil, fmt.Errorf("open file for hash: %w", err) } defer f.Close() @@ -61,3 +72,22 @@ func selectHashFunction(meta *data.TargetFileMeta) (hash.Hash, []byte, error) { return nil, nil, fmt.Errorf("no matching hash function found: %v", meta.HashAlgorithms()) } + +// readCachedHash reads a cached hash from a .sha512 file +// created during packaging when the tar.gz was removed to save space. +func readCachedHash(tarGzPath string, meta *data.TargetFileMeta) ([]byte, error) { + // Check if TUF metadata has SHA512 (currently the only hash file used) + for hashName := range meta.Hashes { + if hashName == "sha512" { + hashPath := tarGzPath + ".sha512" + var hashHex []byte + var err error + if hashHex, err = os.ReadFile(hashPath); err != nil { + return nil, err + } + return hex.DecodeString(strings.TrimSpace(string(hashHex))) + } + } + + return nil, fmt.Errorf("no cached hash file found for %s", tarGzPath) +} diff --git a/orbit/pkg/update/update.go b/orbit/pkg/update/update.go index cb00994342..0e89200fec 100644 --- a/orbit/pkg/update/update.go +++ b/orbit/pkg/update/update.go @@ -463,6 +463,8 @@ func (u *Updater) get(target string) (*LocalTarget, error) { return nil, fmt.Errorf("download %q: %w", repoPath, err) } if strings.HasSuffix(localTarget.Path, ".tar.gz") { + // Remove cached hash files since we have a real tar.gz now + removeCachedHashes(localTarget.Path) if err := os.RemoveAll(localTarget.DirPath); err != nil { return nil, fmt.Errorf("failed to remove old extracted dir: %q: %w", localTarget.DirPath, err) } @@ -476,9 +478,39 @@ func (u *Updater) get(target string) (*LocalTarget, error) { log.Debug().Str("path", localTarget.Path).Str("target", target).Msg("found expected target locally") } case errors.Is(err, os.ErrNotExist): - log.Debug().Err(err).Msg("stat file") - if err := u.download(target, repoPath, localTarget.Path, localTarget.Info.CustomCheckExec); err != nil { - return nil, fmt.Errorf("download %q: %w", repoPath, err) + // Check if we have a cached hash file for tar.gz files + if strings.HasSuffix(localTarget.Path, ".tar.gz") { + hashPath := localTarget.Path + ".sha512" + if _, hashErr := os.Stat(hashPath); hashErr == nil { + // We have a hash file, so check if it matches TUF metadata + meta, err := u.Lookup(target) + if err != nil { + return nil, err + } + if err := checkFileHash(meta, localTarget.Path); err != nil { + // Hash doesn't match or can't be verified, download the tar.gz + log.Debug().Str("info", err.Error()).Msg("hash mismatch or verification failed, downloading") + if err := u.download(target, repoPath, localTarget.Path, localTarget.Info.CustomCheckExec); err != nil { + return nil, fmt.Errorf("download %q: %w", repoPath, err) + } + removeCachedHashes(localTarget.Path) + } else { + // Hash matches! We can proceed without the tar.gz + log.Debug().Str("path", localTarget.Path).Msg("using cached hash, tar.gz not needed") + } + } else { + // No hash file either, need to download + log.Debug().Err(err).Msg("no tar.gz or hash file, downloading") + if err := u.download(target, repoPath, localTarget.Path, localTarget.Info.CustomCheckExec); err != nil { + return nil, fmt.Errorf("download %q: %w", repoPath, err) + } + } + } else { + // Not a tar.gz, just download it + log.Debug().Err(err).Msg("stat file") + if err := u.download(target, repoPath, localTarget.Path, localTarget.Info.CustomCheckExec); err != nil { + return nil, fmt.Errorf("download %q: %w", repoPath, err) + } } if strings.HasSuffix(localTarget.Path, ".pkg") && runtime.GOOS == "darwin" { if err := installPKG(localTarget.Path); err != nil { @@ -493,8 +525,16 @@ func (u *Updater) get(target string) (*LocalTarget, error) { s, err := os.Stat(localTarget.ExecPath) switch { case err == nil: - // OK + // OK - executable exists case errors.Is(err, os.ErrNotExist): + // Check if tar.gz exists before trying to extract + if _, tarErr := os.Stat(localTarget.Path); tarErr != nil { + // No tar.gz to extract from. + // The executable should already be in the initial package, and this error should never happen under normal circumstances. + // Delete the .sha512 file so next run will download the tar.gz + removeCachedHashes(localTarget.Path) + return nil, fmt.Errorf("executable not found and no tar.gz to extract: %q", localTarget.ExecPath) + } if err := extractTarGz(localTarget.Path); err != nil { return nil, fmt.Errorf("extract %q: %w", localTarget.Path, err) } @@ -741,6 +781,13 @@ func extractTarGz(path string) error { } } +// removeCachedHashes removes the .sha512 file that was created +// during packaging to cache the hash when the tar.gz was removed. +func removeCachedHashes(tarGzPath string) { + // Remove hash file, ignore errors (file may not exist) + _ = os.Remove(tarGzPath + ".sha512") +} + func installPKG(path string) error { cmd := exec.Command("installer", "-pkg", path, "-target", "/") if out, err := cmd.CombinedOutput(); err != nil {