Fixed macOS MSI package -- using local wine and wix (#16307)

New flow for `fleetctl --package --type=msi` on macOS using arm64
processor (M1, M2, etc.)
- wine must be installed locally. See
./orbit/tools/build/install-wine-macos.sh and
https://wiki.winehq.org/MacOS for reference.
- --local-wix-dir can be used to point to a local Wix3 installation
(using this switch requires a current Fleet EE subscription)
#15463 

PR for docs: https://github.com/fleetdm/fleet/pull/16459

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

<!-- Note that API documentation changes are now addressed by the
product design team. -->

- [x] Changes file added for user-visible changes in `changes/` or
`orbit/changes/`.
See [Changes
files](https://fleetdm.com/docs/contributing/committing-changes#changes-files)
for more information.
- [x] Manual QA for all new/changed functionality

---------

Co-authored-by: Noah Talerman <47070608+noahtalerman@users.noreply.github.com>

.github/workflows/test-packaging.yml

@@ -77,7 +77,17 @@ jobs:
     - name: Checkout Code
       uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
 
-    # It seems faster not to cache Go dependencies
+    - name: Install wine and wix
+      if: matrix.os == 'macos-latest'
+      run: |
+        ./scripts/macos-install-wine.sh
+        wget https://github.com/wixtoolset/wix3/releases/download/wix3112rtm/wix311-binaries.zip -nv -O wix.zip
+        mkdir wix
+        unzip wix.zip -d wix
+        rm -f wix.zip
+        echo wix installed at $(pwd)/wix
+
+      # It seems faster not to cache Go dependencies
     - name: Install Go Dependencies
       run: make deps-go
 
@@ -107,3 +117,7 @@ jobs:
 
     - name: Build PKG with Fleet Desktop
       run: ./build/fleetctl package --type pkg --enroll-secret=foo --fleet-url=https://localhost:8080 --fleet-desktop
+
+    - name: Build MSI (using local Wix)
+      if: matrix.os == 'macos-latest'
+      run: ./build/fleetctl package --type msi --enroll-secret=foo --fleet-url=https://localhost:8080 --fleet-desktop --local-wix-dir ./wix

changes/15463-msi-on-mac

@@ -0,0 +1,3 @@
+New flow for `fleetctl package --type=msi` on macOS using arm64 processor (M1, M2, etc.)
+- wine must be installed locally. See ./orbit/tools/build/install-wine-macos.sh and https://wiki.winehq.org/MacOS for reference.
+- --local-wix-dir can be used to point to a local Wix3 installation (using this switch requires a current Fleet EE subscription)

cmd/fleetctl/package.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
-	"strings"
 	"time"
 
 	eefleetctl "github.com/fleetdm/fleet/v4/ee/fleetctl"
@@ -289,8 +288,9 @@ func packageCommand() *cli.Command {
 				return errors.New("native tooling is only available in Linux")
 			}
 
-			if opt.LocalWixDir != "" && runtime.GOOS != "windows" {
-				return errors.New(`Could not use local WiX to generate an osquery installer. This option is only available on Windows.
+			if opt.LocalWixDir != "" && runtime.GOOS != "windows" && runtime.GOOS != "darwin" {
+				return errors.New(
+					`Could not use local WiX to generate an osquery installer. This option is only available on Windows and macOS.
 				Visit https://wixtoolset.org/ for more information about how to use WiX.`)
 			}
 
@@ -340,25 +340,8 @@ func packageCommand() *cli.Command {
 				zlog.Logger = zerolog.Nop()
 			}
 
-			const maxAttempts = 9 // see #5732
-			var (
-				attempts int
-				path     string
-				err      error
-			)
-			for attempts < maxAttempts {
-				attempts++
-
-				if attempts > 1 {
-					fmt.Printf("Generating your osquery installer [attempt %d/%d]...\n\n", attempts, maxAttempts)
-				} else {
-					fmt.Println("Generating your osquery installer...")
-				}
-				path, err = buildFunc(opt)
-				if err == nil || !shouldRetry(c.String("type"), opt, err) {
-					break
-				}
-			}
+			fmt.Println("Generating your osquery installer...")
+			path, err := buildFunc(opt)
 			if err != nil {
 				return err
 			}
@@ -379,25 +362,6 @@ To add other devices to Fleet, distribute this installer using Chef, Ansible, Ja
 	}
 }
 
-func shouldRetry(pkgType string, opt packaging.Options, err error) bool {
-	if pkgType != "msi" || runtime.GOOS != "darwin" || runtime.GOARCH != "arm64" {
-		return false
-	}
-
-	// building an MSI on macos M1, check if the error is one that should be retried
-	errStr := err.Error()
-	switch {
-	case strings.Contains(errStr, "package root files: heat failed"):
-		return true
-	case strings.Contains(errStr, "build package: candle failed"):
-		return true
-	case strings.Contains(errStr, "build package: light failed"):
-		return true
-	default:
-		return false
-	}
-}
-
 func checkPEMCertificate(path string) error {
 	cert, err := os.ReadFile(path)
 	if err != nil {

ee/fleetctl/local_wix.go

@@ -5,7 +5,7 @@ import "github.com/urfave/cli/v2"
 func LocalWixDirFlag(dest *string) *cli.StringFlag {
 	return &cli.StringFlag{
 		Name:        "local-wix-dir",
-		Usage:       "Use local install of WiX instead of Docker Hub (only available on Windows w/ WiX v3). This functionality is licensed under the Fleet EE License. Usage requires a current Fleet EE subscription.",
+		Usage:       "Use local install of WiX instead of Docker Hub (only available on Windows and macOS w/ WiX v3). This functionality is licensed under the Fleet EE License. Usage requires a current Fleet EE subscription.",
 		Destination: dest,
 	}
 }

orbit/pkg/packaging/windows.go

@@ -1,10 +1,13 @@
 package packaging
 
 import (
+	"archive/zip"
 	"bytes"
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -17,11 +20,14 @@ import (
 	"github.com/fleetdm/fleet/v4/orbit/pkg/packaging/wix"
 	"github.com/fleetdm/fleet/v4/orbit/pkg/update"
 	"github.com/fleetdm/fleet/v4/pkg/file"
+	"github.com/fleetdm/fleet/v4/pkg/fleethttp"
 	"github.com/fleetdm/fleet/v4/pkg/secure"
 	"github.com/josephspurrier/goversioninfo"
 	"github.com/rs/zerolog/log"
 )
 
+const wixDownload = "https://github.com/wixtoolset/wix3/releases/download/wix3112rtm/wix311-binaries.zip"
+
 // BuildMSI builds a Windows .msi.
 // Note: this function is not safe for concurrent use
 func BuildMSI(opt Options) (string, error) {
@@ -149,7 +155,40 @@ func BuildMSI(opt Options) (string, error) {
 		}
 	}
 
-	if err := wix.Heat(tmpDir, opt.NativeTooling, opt.LocalWixDir); err != nil {
+	absWixDir := opt.LocalWixDir
+	wineChecked := false
+
+	// Download wix for macOS running on arm64, unless a local-wix-dir is provided.
+	// We are using native MSI build on macOS arm64, instead of Docker, because the current fleetdm/wix Docker image is unreliable on macOS arm64.
+	// We are looking into creating a new Docker image for macOS arm64.
+	if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && absWixDir == "" {
+		fmt.Println("Detected macOS arm64. fleetctl must use locally installed wine and wix to build the MSI package.")
+
+		// Ensure wine is installed before downloading wix
+		if err = checkWine(false); err != nil {
+			return "", err
+		}
+		wineChecked = true
+
+		fmt.Printf("Downloading wix from %s\n", wixDownload)
+		client := fleethttp.NewClient()
+		absWixDir = filepath.Join(tmpDir, "wix")
+		err = downloadAndExtractZip(client, wixDownload, absWixDir)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	if absWixDir != "" {
+		absWixDir, err = filepath.Abs(absWixDir)
+		if err != nil {
+			return "", fmt.Errorf("could not get filepath from local-wix-dir %s: %w", opt.LocalWixDir, err)
+		}
+		if err = checkWine(wineChecked); err != nil {
+			return "", err
+		}
+	}
+	if err := wix.Heat(tmpDir, opt.NativeTooling, absWixDir); err != nil {
 		return "", fmt.Errorf("package root files: %w", err)
 	}
 
@@ -157,11 +196,11 @@ func BuildMSI(opt Options) (string, error) {
 		return "", fmt.Errorf("transform heat: %w", err)
 	}
 
-	if err := wix.Candle(tmpDir, opt.NativeTooling, opt.LocalWixDir); err != nil {
+	if err := wix.Candle(tmpDir, opt.NativeTooling, absWixDir); err != nil {
 		return "", fmt.Errorf("build package: %w", err)
 	}
 
-	if err := wix.Light(tmpDir, opt.NativeTooling, opt.LocalWixDir); err != nil {
+	if err := wix.Light(tmpDir, opt.NativeTooling, absWixDir); err != nil {
 		return "", fmt.Errorf("build package: %w", err)
 	}
 
@@ -177,6 +216,20 @@ func BuildMSI(opt Options) (string, error) {
 	return filename, nil
 }
 
+func checkWine(wineChecked bool) error {
+	if !wineChecked && runtime.GOOS == "darwin" {
+		// Ensure wine is installed
+		cmd := exec.Command(wix.WineCmd, "--version")
+		if err := cmd.Run(); err != nil {
+			return fmt.Errorf(
+				"%s failed. Is Wine installed? Creating a fleetd agent for Windows (.msi) requires Wine. To install Wine see the script here: https://github.com/fleetdm/fleet/blob/fleet-v4.44.0/scripts/macos-install-wine.sh %w",
+				wix.WineCmd, err,
+			)
+		}
+	}
+	return nil
+}
+
 func writeWixFile(opt Options, rootPath string) error {
 	// PackageInfo is metadata for the pkg
 	path := filepath.Join(rootPath, "main.wxs")
@@ -378,3 +431,102 @@ func writeResourceSyso(opt Options, orbitPath string) error {
 
 	return nil
 }
+
+func downloadAndExtractZip(client *http.Client, urlPath string, destPath string) error {
+	zipFile, err := os.CreateTemp("", "file.zip")
+	if err != nil {
+		return fmt.Errorf("create file: %w", err)
+	}
+	defer zipFile.Close()
+	defer os.Remove(zipFile.Name())
+
+	req, err := http.NewRequest(http.MethodGet, urlPath, nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("could not download %s: %w", urlPath, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("could not download %s: received http status code %s", urlPath, resp.Status)
+	}
+	_, err = io.Copy(zipFile, resp.Body)
+	if err != nil {
+		return fmt.Errorf("could not write %s: %w", zipFile.Name(), err)
+	}
+
+	// Open the downloaded file for reading. With zip, we cannot unzip directly from resp.Body
+	zipReader, err := zip.OpenReader(zipFile.Name())
+	if err != nil {
+		return fmt.Errorf("could not open %s: %w", zipFile.Name(), err)
+	}
+	defer zipReader.Close()
+
+	err = os.MkdirAll(filepath.Dir(destPath), 0755)
+	if err != nil {
+		return fmt.Errorf("could not create directory %s: %w", filepath.Dir(destPath), err)
+	}
+
+	// Extract each file in the archive
+	for _, archiveReader := range zipReader.File {
+		err = extractZipFile(archiveReader, destPath)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+
+}
+
+func extractZipFile(archiveReader *zip.File, destPath string) error {
+	if archiveReader.FileInfo().Mode()&os.ModeSymlink != 0 {
+		// Skip symlinks for security reasons
+		return nil
+	}
+
+	// Open the file in the archive
+	archiveFile, err := archiveReader.Open()
+	if err != nil {
+		return fmt.Errorf("could not open archive %s: %w", archiveReader.Name, err)
+	}
+	defer archiveFile.Close()
+
+	// Clean the archive path to prevent extracting files outside the destination.
+	archivePath := filepath.Clean(archiveReader.Name)
+	if strings.HasPrefix(archivePath, ".."+string(filepath.Separator)) {
+		// Skip relative paths for security reasons
+		return nil
+	}
+	// Prepare to write the file
+	finalPath := filepath.Join(destPath, archivePath)
+
+	// Check if the file to extract is just a directory
+	if archiveReader.FileInfo().IsDir() {
+		err = os.MkdirAll(finalPath, 0755)
+		if err != nil {
+			return fmt.Errorf("could not create directory %s: %w", finalPath, err)
+		}
+	} else {
+		// Create all needed directories
+		if os.MkdirAll(filepath.Dir(finalPath), 0755) != nil {
+			return fmt.Errorf("could not create directory %s: %w", filepath.Dir(finalPath), err)
+		}
+
+		// Prepare to write the destination file
+		destinationFile, err := os.OpenFile(finalPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, archiveReader.Mode())
+		if err != nil {
+			return fmt.Errorf("could not open file %s: %w", finalPath, err)
+		}
+		defer destinationFile.Close()
+
+		// Write the destination file
+		if _, err = io.Copy(destinationFile, archiveFile); err != nil {
+			return fmt.Errorf("could not write file %s: %w", finalPath, err)
+		}
+	}
+	return nil
+}

orbit/pkg/packaging/windows_test.go

@@ -6,6 +6,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/fleetdm/fleet/v4/pkg/fleethttp"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -89,3 +91,14 @@ func TestSanitizeVersion(t *testing.T) {
 		require.Equal(t, tC.Parts, result)
 	}
 }
+
+func TestDownloadAndExtractZip(t *testing.T) {
+	t.Parallel()
+	path := t.TempDir()
+	client := fleethttp.NewClient()
+	err := downloadAndExtractZip(client, wixDownload, path)
+	require.NoError(t, err)
+	assert.FileExists(t, filepath.Join(path, "heat.exe"))
+	assert.FileExists(t, filepath.Join(path, "candle.exe"))
+	assert.FileExists(t, filepath.Join(path, "light.exe"))
+}

orbit/pkg/packaging/wix/wix.go

@@ -7,10 +7,15 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"runtime"
 )
 
 const (
 	directoryReference = "ORBITROOT"
+	imageName          = "fleetdm/wix:latest"
+	dockerPlatform     = "linux/amd64"
+	WineCmd            = "wine64"
 )
 
 // Heat runs the WiX Heat command on the provided directory.
@@ -24,15 +29,18 @@ func Heat(path string, native bool, localWixDir string) error {
 	if !native && localWixDir == "" {
 		args = append(
 			args,
-			"docker", "run", "--rm", "--platform", "linux/amd64",
+			"docker", "run", "--rm", "--platform", dockerPlatform,
 			"--volume", path+":/wix", // mount volume
-			"fleetdm/wix:latest", // image name
+			imageName, // image name
 		)
 	}
 
 	heatPath := `heat`
 	if localWixDir != "" {
-		heatPath = localWixDir + `\heat.exe`
+		heatPath = filepath.Join(localWixDir, `heat.exe`)
+		if runtime.GOOS == "darwin" {
+			args = append(args, WineCmd)
+		}
 	}
 
 	args = append(args,
@@ -48,6 +56,10 @@ func Heat(path string, native bool, localWixDir string) error {
 	cmd := exec.Command(args[0], args[1:]...)
 	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
 
+	if args[0] == WineCmd {
+		cmd.Env = append(os.Environ(), "WINEDEBUG=-all")
+	}
+
 	if native || localWixDir != "" {
 		cmd.Dir = path
 	}
@@ -69,15 +81,18 @@ func Candle(path string, native bool, localWixDir string) error {
 	if !native && localWixDir == "" {
 		args = append(
 			args,
-			"docker", "run", "--rm", "--platform", "linux/amd64",
+			"docker", "run", "--rm", "--platform", dockerPlatform,
 			"--volume", path+":/wix", // mount volume
-			"fleetdm/wix:latest", // image name
+			imageName, // image name
 		)
 	}
 
 	candlePath := `candle`
 	if localWixDir != "" {
-		candlePath = localWixDir + `\candle.exe`
+		candlePath = filepath.Join(localWixDir, `candle.exe`)
+		if runtime.GOOS == "darwin" {
+			args = append(args, WineCmd)
+		}
 	}
 	args = append(args,
 		candlePath, "heat.wxs", "main.wxs", // command
@@ -88,6 +103,10 @@ func Candle(path string, native bool, localWixDir string) error {
 	cmd := exec.Command(args[0], args[1:]...)
 	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
 
+	if args[0] == WineCmd {
+		cmd.Env = append(os.Environ(), "WINEDEBUG=-all")
+	}
+
 	if native || localWixDir != "" {
 		cmd.Dir = path
 	}
@@ -109,15 +128,18 @@ func Light(path string, native bool, localWixDir string) error {
 	if !native && localWixDir == "" {
 		args = append(
 			args,
-			"docker", "run", "--rm", "--platform", "linux/amd64",
+			"docker", "run", "--rm", "--platform", dockerPlatform,
 			"--volume", path+":/wix", // mount volume
-			"fleetdm/wix:latest", // image name
+			imageName, // image name
 		)
 	}
 
 	lightPath := `light`
 	if localWixDir != "" {
-		lightPath = localWixDir + `\light.exe`
+		lightPath = filepath.Join(localWixDir, `light.exe`)
+		if runtime.GOOS == "darwin" {
+			args = append(args, WineCmd)
+		}
 	}
 	args = append(args,
 		lightPath, "heat.wixobj", "main.wixobj", // command
@@ -130,6 +152,10 @@ func Light(path string, native bool, localWixDir string) error {
 	cmd := exec.Command(args[0], args[1:]...)
 	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
 
+	if args[0] == WineCmd {
+		cmd.Env = append(os.Environ(), "WINEDEBUG=-all")
+	}
+
 	if native || localWixDir != "" {
 		cmd.Dir = path
 	}

scripts/macos-install-wine.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+# Reference: https://wiki.winehq.org/MacOS
+# NOTE: Wine can be installed without brew via a distribution such as https://github.com/Gcenx/macOS_Wine_builds/releases/tag/9.0, or by building from source.
+
+# Check if brew is installed
+if ! command -v brew >/dev/null 2>&1 ; then
+    echo "Homebrew is not installed. Please install Homebrew first. For instructions, see https://brew.sh/"
+    exit 1
+fi
+
+# Install wine via brew
+brew install --cask --no-quarantine https://raw.githubusercontent.com/Homebrew/homebrew-cask/1ecfe82f84e0f3c3c6b741d3ddc19a164c2cb18d/Casks/w/wine-stable.rb