diff --git a/.github/workflows/test-packaging.yml b/.github/workflows/test-packaging.yml index c8d9f109e0..29d52da08f 100644 --- a/.github/workflows/test-packaging.yml +++ b/.github/workflows/test-packaging.yml @@ -1,11 +1,10 @@ -# This workflow tests packaging of fleetd with the -# `fleetctl package` command. +# This workflow tests packaging of fleetd with the `fleetctl package` command +# on Linux (ubuntu-latest). # # It fetches the targets: orbit, osquery and fleet-desktop from the default # (Fleet's) TUF server, https://tuf.fleetctl.com. # -# Docker and colima are extremely unreliable on macOS Github runners -# thus this workflow is not testing MSI package generation on macOS. +# All package types (deb, rpm, pkg.tar.zst, msi, pkg) are built on ubuntu-latest. name: Test packaging on: @@ -22,7 +21,6 @@ on: - "ee/fleetctl/**.go" - "tools/fleetctl-docker/**" - "tools/wix-docker/**" - - "tools/bomutils-docker/**" - ".github/workflows/test-packaging.yml" pull_request: paths: @@ -33,7 +31,6 @@ on: - "ee/fleetctl/**.go" - "tools/fleetctl-docker/**" - "tools/wix-docker/**" - - "tools/bomutils-docker/**" - ".github/workflows/test-packaging.yml" workflow_dispatch: # Manual @@ -55,7 +52,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-4core, macos-15, macos-26] + os: [ubuntu-latest] runs-on: ${{ matrix.os }} steps: @@ -65,17 +62,11 @@ jobs: egress-policy: audit - name: Pull fleetdm/wix - if: ${{ !startsWith(matrix.os, 'macos') }} - # Run in background while other steps complete to speed up the workflow + # Run in background while other steps complete to speed up the workflow. + # Only MSI generation needs a Docker image now that pkg is pure Go. run: | docker pull fleetdm/wix:latest & - - name: Pull fleetdm/bomutils - if: ${{ !startsWith(matrix.os, 'macos') }} - # Run in background while other steps complete to speed up the workflow - run: | - docker pull fleetdm/bomutils:latest & - - name: Checkout Code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 with: @@ -108,11 +99,9 @@ jobs: run: ./build/fleetctl package --type pkg.tar.zst --enroll-secret=foo --fleet-url=https://localhost:8080 --fleet-desktop - name: Build MSI - if: ${{ !startsWith(matrix.os, 'macos') }} run: ./build/fleetctl package --type msi --enroll-secret=foo --fleet-url=https://localhost:8080 - name: Build MSI with Fleet Desktop - if: ${{ !startsWith(matrix.os, 'macos') }} run: ./build/fleetctl package --type msi --enroll-secret=foo --fleet-url=https://localhost:8080 --fleet-desktop - name: Build PKG diff --git a/changes/48448-remove-bomutils-docker b/changes/48448-remove-bomutils-docker new file mode 100644 index 0000000000..4483e93bd9 --- /dev/null +++ b/changes/48448-remove-bomutils-docker @@ -0,0 +1 @@ +- Removed the `fleetdm/bomutils` Docker dependency for generating macOS `.pkg` fleetd installers; the Bill of Materials and xar archive are now written by pure-Go code, so `fleetctl package --type pkg` no longer requires Docker, `mkbom`, or `xar`. diff --git a/orbit/pkg/packaging/bom.go b/orbit/pkg/packaging/bom.go new file mode 100644 index 0000000000..cc19972149 --- /dev/null +++ b/orbit/pkg/packaging/bom.go @@ -0,0 +1,406 @@ +package packaging + +import ( + "bytes" + "encoding/binary" + "fmt" + "os" + "path/filepath" + "sort" +) + +// This file implements a minimal, pure-Go writer for the macOS BOM (Bill of +// Materials) format, simulating the external `mkbom`/`lsbom` macOS tools. +// +// A BOM is a block store: +// +// [ 32-byte header ][ blocks... ][ vars ][ block table (index) ] +// +// The header locates the block table (an array of (offset,length) pointers, one +// per block index) and the vars section (named -> block index). Named variables +// point at the top-level structures: BomInfo, Paths, HLIndex, VIndex, Size64. +// +// "Paths" is a B-tree whose single leaf lists (PathInfo1, File) block-index +// pairs, one per path. PathInfo1 -> PathInfo2 holds the metadata (type, mode, +// uid/gid, size, checksum); File holds the parent path id and the base name. +// Ownership is fixed to root/admin (0/80), matching the previous mkbom -u0 -g80 +// behavior. Per-file checksums use the POSIX cksum (CRC-32/CKSUM) algorithm, +// exactly as Apple's mkbom records them. +// +// This writer does not reproduce Apple's exact block layout byte-for-byte (its +// block table is pre-sized with a free list); it produces a compact, valid BOM +// that lsbom and the macOS Installer read identically. Byte-identical output was +// never a requirement -- an identical lsbom manifest is. + +// bomChecksumTable is the CRC-32 table for polynomial 0x04C11DB7 (MSB-first), +// used by the POSIX cksum algorithm. +var bomChecksumTable = func() [256]uint32 { + var t [256]uint32 + for i := range t { + c := uint32(i) << 24 + for range 8 { + if c&0x80000000 != 0 { + c = (c << 1) ^ 0x04C11DB7 + } else { + c <<= 1 + } + } + t[i] = c + } + return t +}() + +// bomChecksum computes the POSIX cksum (CRC-32/CKSUM) of data: the CRC-32 over +// the data followed by the little-endian minimal-byte encoding of its length, +// finally inverted. This matches the checksum Apple's mkbom stores per file. +func bomChecksum(data []byte) uint32 { + var crc uint32 + for _, b := range data { + crc = (crc << 8) ^ bomChecksumTable[byte(crc>>24)^b] + } + for n := len(data); n != 0; n >>= 8 { + crc = (crc << 8) ^ bomChecksumTable[byte(crc>>24)^byte(n)] + } + return ^crc +} + +// bomPath is one entry in the BOM path tree. +type bomPath struct { + id uint32 + parentID uint32 // 0 for the root "." + name string // base name; "." for the root + isDir bool + mode uint16 // full st_mode (type bits | permissions) + size uint32 + checksum uint32 // POSIX cksum of contents; 0 for directories +} + +// Fixed block indices. Per-path blocks follow, starting at bomFirstPathBlock. +const ( + // Block index 0 is always the null block. + bomInfoBlock = 1 + bomPathsTree = 2 + bomPathsLeaf = 3 + bomHLIndexTree = 4 + bomHLIndexLeaf = 5 + bomVIndexBlock = 6 + bomVIndexTree = 7 + bomVIndexLeaf = 8 + bomSize64Tree = 9 + bomSize64Leaf = 10 + bomFirstPathBlock = 11 +) + +// writeBom walks srcDir and writes a BOM describing its tree to dstPath, with +// all entries owned by root/admin (0/80). +func writeBom(srcDir, dstPath string) error { + paths, err := collectBomPaths(srcDir) + if err != nil { + return fmt.Errorf("collect bom paths: %w", err) + } + + data, err := buildBom(paths) + if err != nil { + return err + } + if err := os.WriteFile(dstPath, data, 0o644); err != nil { + return fmt.Errorf("write bom: %w", err) + } + return nil +} + +// collectBomPaths walks srcDir depth-first (children sorted by name), returning +// path entries with sequential ids assigned in that order. The root directory +// itself is recorded as ".". +func collectBomPaths(srcDir string) ([]*bomPath, error) { + info, err := os.Stat(srcDir) + if err != nil { + return nil, err + } + + var ( + out []*bomPath + nextID uint32 = 1 + ) + root := &bomPath{id: nextID, parentID: 0, name: ".", isDir: true, mode: bomUnixMode(info)} + nextID++ + out = append(out, root) + + var walk func(dir string, parentID uint32) error + walk = func(dir string, parentID uint32) error { + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }) + + for _, de := range entries { + // The fleetd payload is built only from files the packaging code + // writes directly and from TUF target tarballs unpacked by + // extractTarGz, which rejects any tar entry that is not a regular + // file or directory. The orbit "current" symlink is created by the + // postinstall script at install time, not shipped in the payload. + // So a symlink (or other special file) should never appear here; + // fail loudly rather than emit a malformed BOM entry (e.g. a type=3 + // symlink with no link target). + if !de.IsDir() && !de.Type().IsRegular() { + return fmt.Errorf("unsupported file type %s for %q", de.Type(), filepath.Join(dir, de.Name())) + } + + fi, err := de.Info() + if err != nil { + return err + } + p := &bomPath{id: nextID, parentID: parentID, name: de.Name(), isDir: de.IsDir(), mode: bomUnixMode(fi)} + nextID++ + + if !de.IsDir() { + contents, err := os.ReadFile(filepath.Join(dir, de.Name())) + if err != nil { + return err + } + p.size = uint32(len(contents)) //nolint:gosec // fleetd payload files are well under 4GB + p.checksum = bomChecksum(contents) + } + out = append(out, p) + + if de.IsDir() { + if err := walk(filepath.Join(dir, de.Name()), p.id); err != nil { + return err + } + } + } + return nil + } + + if err := walk(srcDir, root.id); err != nil { + return nil, err + } + return out, nil +} + +// bomUnixMode returns the full st_mode (file-type bits OR-ed with permission +// bits) for a file, as stored in a BOM. +func bomUnixMode(info os.FileInfo) uint16 { + perm := uint16(info.Mode().Perm()) //nolint:gosec // permission bits fit in uint16 + // collectBomPaths rejects anything that isn't a regular file or directory, + // so only those two types reach here. + if info.IsDir() { + return 0o040000 | perm + } + return 0o100000 | perm +} + +// buildBom assembles the full BOM byte stream for the given path entries. +func buildBom(paths []*bomPath) ([]byte, error) { + n := len(paths) + blocks := make([][]byte, bomFirstPathBlock+3*n) + + // Per-path blocks, laid out as [PathInfo2, File, PathInfo1] per path (the + // same ordering Apple's mkbom uses). + type leafEntry struct { + parentID uint32 + name string + pi1Idx, fileID uint32 + } + entries := make([]leafEntry, 0, n) + for k, p := range paths { + pi2Idx := bomFirstPathBlock + 3*k + fileIdx := pi2Idx + 1 + pi1Idx := pi2Idx + 2 + + blocks[pi2Idx] = buildBomPathInfo2(p) + blocks[fileIdx] = buildBomFile(p) + blocks[pi1Idx] = buildBomPathInfo1(p.id, uint32(pi2Idx)) //nolint:gosec // block index fits uint32 + + entries = append(entries, leafEntry{p.parentID, p.name, uint32(pi1Idx), uint32(fileIdx)}) //nolint:gosec // block indices fit uint32 + } + + // The Paths leaf is a B-tree node keyed by (parent id, name): lsbom and the + // Installer traverse it in key order, so the pairs must be sorted that way. + sort.Slice(entries, func(i, j int) bool { + if entries[i].parentID != entries[j].parentID { + return entries[i].parentID < entries[j].parentID + } + return entries[i].name < entries[j].name + }) + leafPairs := make([][2]uint32, len(entries)) + for i, e := range entries { + leafPairs[i] = [2]uint32{e.pi1Idx, e.fileID} + } + + // Fixed structures. + blocks[bomInfoBlock] = buildBomInfo(uint32(n) + 1) //nolint:gosec // path count fits uint32 + blocks[bomPathsTree] = buildBomTree(bomPathsLeaf, uint32(n), 4096) //nolint:gosec // path count fits uint32 + blocks[bomPathsLeaf] = buildBomLeaf(leafPairs) + blocks[bomHLIndexTree] = buildBomTree(bomHLIndexLeaf, 0, 4096) + blocks[bomHLIndexLeaf] = buildBomLeaf(nil) + blocks[bomVIndexBlock] = buildBomVIndex(bomVIndexTree) + blocks[bomVIndexTree] = buildBomTree(bomVIndexLeaf, 0, 128) + blocks[bomVIndexLeaf] = buildBomLeaf(nil) + blocks[bomSize64Tree] = buildBomTree(bomSize64Leaf, 0, 4096) + blocks[bomSize64Leaf] = buildBomLeaf(nil) + + // Lay out block data after the 32-byte header, recording each block's + // address. The null block (index 0) has address 0 and length 0. + addrs := make([]uint32, len(blocks)) + var body bytes.Buffer + cursor := uint32(32) + for i := 1; i < len(blocks); i++ { + addrs[i] = cursor + body.Write(blocks[i]) + cursor += uint32(len(blocks[i])) //nolint:gosec // block sizes are small + } + + // Vars section, then the block table (index). + vars := buildBomVars() + varsOffset := cursor + cursor += uint32(len(vars)) //nolint:gosec // vars section is tiny + + index := buildBomIndex(blocks, addrs) + indexOffset := cursor + + var out bytes.Buffer + out.WriteString("BOMStore") + be := binary.BigEndian + writeU32 := func(v uint32) { _ = binary.Write(&out, be, v) } + writeU32(1) // version + writeU32(uint32(len(blocks) - 1)) //nolint:gosec // number of non-null blocks + writeU32(indexOffset) // indexOffset + writeU32(uint32(len(index))) //nolint:gosec // indexLength + writeU32(varsOffset) // varsOffset + writeU32(uint32(len(vars))) //nolint:gosec // varsLength + out.Write(body.Bytes()) + out.Write(vars) + out.Write(index) + return out.Bytes(), nil +} + +// buildBomPathInfo2 renders the metadata block for a path (35 bytes for files, +// 31 for directories). Ownership is fixed to uid 0 / gid 80 (root/admin). +func buildBomPathInfo2(p *bomPath) []byte { + var b bytes.Buffer + be := binary.BigEndian + typ := byte(1) // regular file + if p.isDir { + typ = 2 // directory + } + b.WriteByte(typ) + b.WriteByte(1) // unknown0 (always 1) + _ = binary.Write(&b, be, uint16(3)) // architecture + _ = binary.Write(&b, be, p.mode) + _ = binary.Write(&b, be, uint32(0)) // uid = root + _ = binary.Write(&b, be, uint32(80)) // gid = admin + _ = binary.Write(&b, be, uint32(0)) // mtime (0, matching mkbom -i) + _ = binary.Write(&b, be, p.size) + b.WriteByte(1) // unknown1 (always 1) + _ = binary.Write(&b, be, p.checksum) + _ = binary.Write(&b, be, uint32(0)) // linkNameLength (no symlink targets in our payload) + if !p.isDir { + _ = binary.Write(&b, be, uint32(0)) // trailing reserved word present only on files + } + return b.Bytes() +} + +// buildBomFile renders a File block: parent path id followed by the NUL- +// terminated base name. +func buildBomFile(p *bomPath) []byte { + var b bytes.Buffer + _ = binary.Write(&b, binary.BigEndian, p.parentID) + b.WriteString(p.name) + b.WriteByte(0) + return b.Bytes() +} + +// buildBomPathInfo1 renders a PathInfo1 block: the path id and the block index +// of its PathInfo2. +func buildBomPathInfo1(id, pathInfo2Block uint32) []byte { + var b bytes.Buffer + _ = binary.Write(&b, binary.BigEndian, id) + _ = binary.Write(&b, binary.BigEndian, pathInfo2Block) + return b.Bytes() +} + +// buildBomTree renders a "tree" block pointing at its (single) child leaf. +func buildBomTree(childBlock, pathCount, blockSize uint32) []byte { + var b bytes.Buffer + be := binary.BigEndian + b.WriteString("tree") + _ = binary.Write(&b, be, uint32(1)) // version + _ = binary.Write(&b, be, childBlock) + _ = binary.Write(&b, be, blockSize) + _ = binary.Write(&b, be, pathCount) + b.WriteByte(0) // unknown + return b.Bytes() +} + +// buildBomLeaf renders a B-tree leaf listing (index0, index1) pairs. +func buildBomLeaf(pairs [][2]uint32) []byte { + var b bytes.Buffer + be := binary.BigEndian + _ = binary.Write(&b, be, uint16(1)) // isLeaf + _ = binary.Write(&b, be, uint16(len(pairs))) //nolint:gosec // pair count fits uint16 + _ = binary.Write(&b, be, uint32(0)) // forward + _ = binary.Write(&b, be, uint32(0)) // backward + for _, pr := range pairs { + _ = binary.Write(&b, be, pr[0]) + _ = binary.Write(&b, be, pr[1]) + } + return b.Bytes() +} + +// buildBomVIndex renders the VIndex wrapper: {version, tree block index, flag}. +func buildBomVIndex(treeBlock uint32) []byte { + var b bytes.Buffer + _ = binary.Write(&b, binary.BigEndian, uint32(1)) + _ = binary.Write(&b, binary.BigEndian, treeBlock) + b.WriteByte(0) + return b.Bytes() +} + +// buildBomInfo renders the BomInfo block. +func buildBomInfo(numPaths uint32) []byte { + var b bytes.Buffer + be := binary.BigEndian + _ = binary.Write(&b, be, uint32(1)) // version + _ = binary.Write(&b, be, numPaths) + _ = binary.Write(&b, be, uint32(0)) // numberOfInfoEntries + return b.Bytes() +} + +// buildBomVars renders the named variables pointing at the top-level blocks. +func buildBomVars() []byte { + vars := []struct { + name string + block uint32 + }{ + {"BomInfo", bomInfoBlock}, + {"Paths", bomPathsTree}, + {"HLIndex", bomHLIndexTree}, + {"VIndex", bomVIndexBlock}, + {"Size64", bomSize64Tree}, + } + var b bytes.Buffer + be := binary.BigEndian + _ = binary.Write(&b, be, uint32(len(vars))) //nolint:gosec // small fixed count + for _, v := range vars { + _ = binary.Write(&b, be, v.block) + b.WriteByte(byte(len(v.name))) //nolint:gosec // var names are short constants + b.WriteString(v.name) + } + return b.Bytes() +} + +// buildBomIndex renders the block table: a pointer (offset, length) per block +// index, followed by an empty free list. +func buildBomIndex(blocks [][]byte, addrs []uint32) []byte { + var b bytes.Buffer + be := binary.BigEndian + _ = binary.Write(&b, be, uint32(len(blocks))) //nolint:gosec // block count fits uint32 + for i := range blocks { + _ = binary.Write(&b, be, addrs[i]) + _ = binary.Write(&b, be, uint32(len(blocks[i]))) //nolint:gosec // block sizes are small + } + _ = binary.Write(&b, be, uint32(0)) // free-list count + return b.Bytes() +} diff --git a/orbit/pkg/packaging/bom_darwin_test.go b/orbit/pkg/packaging/bom_darwin_test.go new file mode 100644 index 0000000000..aa6fe906ec --- /dev/null +++ b/orbit/pkg/packaging/bom_darwin_test.go @@ -0,0 +1,85 @@ +//go:build darwin + +package packaging + +import ( + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestWriteBomMatchesMkbom builds a BOM two ways for the same tree -- via the +// native mkbom pipeline used by xarBom (mkbom -> lsbom -> 0/80 transform -> +// mkbom -i) and via the pure-Go writeBom -- then asserts lsbom reports an +// identical manifest for both. This is the functional-equivalence bar for the +// mkbom replacement. +func TestWriteBomMatchesMkbom(t *testing.T) { + for _, tool := range []string{"mkbom", "lsbom"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s not available", tool) + } + } + + root := t.TempDir() + // A representative tree: nested dirs, an empty file, a binary-ish file, a + // name with a space, and varied permissions. + writeFile(t, filepath.Join(root, "opt", "orbit", "secret.txt"), []byte("SUPERSECRET"), 0o600) + writeFile(t, filepath.Join(root, "opt", "orbit", "osquery.flags"), []byte{}, 0o600) + writeFile(t, filepath.Join(root, "opt", "orbit", "bin", "orbit"), []byte("\x7fELF binary-ish payload"), 0o755) + writeFile(t, filepath.Join(root, "Library", "LaunchDaemons", "com.fleetdm.orbit.plist"), []byte("\n"), 0o644) + writeFile(t, filepath.Join(root, "opt", "orbit", "bin", "desktop", "Fleet Desktop.app", "Contents", "Info.plist"), []byte(""), 0o644) + + // Reference BOM via the native pipeline (mirrors xarBom's darwin branch). + refBom := filepath.Join(root, "..", "ref.bom") + inBom := filepath.Join(t.TempDir(), "inBom") + require.NoError(t, exec.Command("mkbom", root, inBom).Run()) //nolint:gosec + lsOut, err := exec.Command("lsbom", inBom).Output() //nolint:gosec + require.NoError(t, err) + // Rewrite ownership to root/admin (0/80), as the old darwin pipeline did. + transformed := regexp.MustCompile(`(.+)\t([0-9]+/[0-9]+)`).ReplaceAll(lsOut, []byte("$1\t0/80")) + require.NoError(t, os.WriteFile(inBom, transformed, 0o644)) + cmd := exec.Command("mkbom", "-i", inBom, refBom) //nolint:gosec + require.NoError(t, cmd.Run()) + + // Pure-Go BOM. + myBom := filepath.Join(t.TempDir(), "my.bom") + require.NoError(t, writeBom(root, myBom)) + + require.Equal(t, sortedLsbom(t, refBom), sortedLsbom(t, myBom), + "lsbom manifest of writeBom output must match the native mkbom pipeline") +} + +// TestWriteBomRejectsSymlink verifies writeBom fails loudly on a symlink rather +// than emitting a malformed BOM entry. Symlinks cannot legitimately appear in a +// fleetd payload (extractTarGz rejects them; the orbit "current" symlink is +// created by postinstall at install time), so this is a defensive guard. +func TestWriteBomRejectsSymlink(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "real.txt"), []byte("hi"), 0o644) + require.NoError(t, os.Symlink("real.txt", filepath.Join(root, "link.txt"))) + + err := writeBom(root, filepath.Join(t.TempDir(), "out.bom")) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported file type") +} + +func writeFile(t *testing.T, path string, data []byte, mode os.FileMode) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, data, mode)) +} + +func sortedLsbom(t *testing.T, bom string) string { + t.Helper() + out, err := exec.Command("lsbom", bom).Output() //nolint:gosec + require.NoErrorf(t, err, "lsbom failed to read %s", bom) + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + sort.Strings(lines) + return strings.Join(lines, "\n") +} diff --git a/orbit/pkg/packaging/macos.go b/orbit/pkg/packaging/macos.go index ea3fd0e66e..c2c4a142d4 100644 --- a/orbit/pkg/packaging/macos.go +++ b/orbit/pkg/packaging/macos.go @@ -8,7 +8,6 @@ import ( "os" "os/exec" "path/filepath" - "regexp" "runtime" "github.com/Masterminds/semver" @@ -19,10 +18,6 @@ import ( "github.com/rs/zerolog/log" ) -var bomRegexp = regexp.MustCompile(`(.+)\t([0-9]+/[0-9]+)`) - -// See helful docs in http://bomutils.dyndns.org/tutorial.html - // BuildPkg builds a macOS .pkg. // // Building packages works out of the box in macOS, but it's also supported on @@ -148,7 +143,7 @@ func BuildPkg(opt Options) (string, error) { // Build package - if err := xarBom(opt, tmpDir); err != nil { + if err := xarBom(tmpDir); err != nil { return "", fmt.Errorf("build pkg: %w", err) } @@ -331,12 +326,8 @@ func writeUpdateClientCertificate(opt Options, orbitRoot string) error { return nil } -// xarBom creates the actual .pkg format. It's a xar archive with a BOM (Bill of -// materials?). See http://bomutils.dyndns.org/tutorial.html. -func xarBom(opt Options, rootPath string) error { - // Adapted from BSD licensed - // https://github.com/go-flutter-desktop/hover/blob/v0.46.2/cmd/packaging/darwin-pkg.go - +// xarBom creates the actual .pkg format. It's a xar archive with a BOM (Bill of materials). +func xarBom(rootPath string) error { // Copy payload/scripts if err := cpio( filepath.Join(rootPath, "root"), @@ -351,103 +342,20 @@ func xarBom(opt Options, rootPath string) error { return fmt.Errorf("cpio Scripts: %w", err) } - // Make Bill of materials (bom) - var cmdMkbom *exec.Cmd - isDarwin := runtime.GOOS == "darwin" - isLinuxNative := runtime.GOOS == "linux" && opt.NativeTooling - - switch { - case isDarwin: - // Using mkbom directly results in permissions listed for the current user and group. We - // transform the output in order to explicitly set root (0) and admin (80). - inBomPath := filepath.Join(rootPath, "inBom") - cmd := exec.Command("mkbom", filepath.Join(rootPath, "root"), inBomPath) - if err := cmd.Run(); err != nil { - return fmt.Errorf("initial mkbom: %w", err) - } - bomContents, err := exec.Command("lsbom", inBomPath).Output() - if err != nil { - return fmt.Errorf("lsbom inBom: %w", err) - } - bomContents = bomReplace(bomContents) - if err := os.WriteFile(inBomPath, bomContents, 0); err != nil { - return fmt.Errorf("write inBom: %w", err) - } - - // Use the file list (with transformed permissions) via -i flag - cmdMkbom = exec.Command("mkbom", "-i", "inBom", filepath.Join("flat", "base.pkg", "Bom")) - cmdMkbom.Dir = rootPath - - // No need for transformation when using the Linux mkbom because of the -u and -g flags - // available in that command. - case isLinuxNative: - cmdMkbom = exec.Command( - "mkbom", "-u", "0", "-g", "80", - filepath.Join(rootPath, "root"), filepath.Join("flat", "base.pkg", "Bom"), - ) - cmdMkbom.Dir = rootPath - default: - // Same as linux native, but modified for running in Docker. This should - // be either Windows, or Linux without the --native-tooling flag. - cmdMkbom = exec.Command( - "docker", "run", "--rm", "-v", rootPath+":/root", "fleetdm/bomutils", - "mkbom", "-u", "0", "-g", "80", - // Use / instead of filepath.Join because these will always be paths within the Docker - // container (so Linux file paths) -- if we use filepath.Join we'll get invalid paths on - // Windows due to use of backslashes. - "/root/root", "/root/flat/base.pkg/Bom", - ) + if err := writeBom( + filepath.Join(rootPath, "root"), + filepath.Join(rootPath, "flat", "base.pkg", "Bom"), + ); err != nil { + return fmt.Errorf("write bom: %w", err) } - cmdMkbom.Stdout, cmdMkbom.Stderr = os.Stdout, os.Stderr - if err := cmdMkbom.Run(); err != nil { - return fmt.Errorf("mkbom: %w", err) - } - - // List files for xar - var files []string - err := filepath.Walk( - filepath.Join(rootPath, "flat"), - func(path string, info os.FileInfo, _ error) error { - relativePath, err := filepath.Rel(filepath.Join(rootPath, "flat"), path) - if err != nil { - return err - } - files = append(files, relativePath) - return nil - }, - ) - if err != nil { - return fmt.Errorf("iterate files: %w", err) - } - - // Make xar - var cmdXar *exec.Cmd - switch { - case isDarwin, isLinuxNative: - cmdXar = exec.Command("xar", append([]string{"--compression", "none", "-cf", filepath.Join("..", "orbit.pkg")}, files...)...) - cmdXar.Dir = filepath.Join(rootPath, "flat") - default: - cmdXar = exec.Command( - "docker", "run", "--rm", "-v", rootPath+":/root", "-w", "/root/flat", "fleetdm/bomutils", - "xar", - ) - cmdXar.Args = append(cmdXar.Args, append([]string{"--compression", "none", "-cf", "/root/orbit.pkg"}, files...)...) - } - - cmdXar.Stdout, cmdXar.Stderr = os.Stdout, os.Stderr - if err := cmdXar.Run(); err != nil { - return fmt.Errorf("run xar: %w", err) + if err := writeXar(filepath.Join(rootPath, "flat"), filepath.Join(rootPath, "orbit.pkg")); err != nil { + return fmt.Errorf("write xar: %w", err) } return nil } -// bomReplace replaces the permission strings (typically "501/20") with the appropriate string ("0/80") -func bomReplace(inBom []byte) []byte { - return bomRegexp.ReplaceAll(inBom, []byte("$1\t0/80")) -} - func cpio(srcPath, dstPath string) error { // This is the compression routine that is expected for pkg files. dst, err := secure.OpenFile(dstPath, os.O_RDWR|os.O_CREATE, 0o755) diff --git a/orbit/pkg/packaging/xar.go b/orbit/pkg/packaging/xar.go new file mode 100644 index 0000000000..0e5c808c24 --- /dev/null +++ b/orbit/pkg/packaging/xar.go @@ -0,0 +1,244 @@ +package packaging + +import ( + "bytes" + "compress/zlib" + "crypto/sha1" //nolint:gosec // xar's on-disk checksum format uses SHA-1; not used for security + "encoding/binary" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// This file implements a minimal, pure-Go writer for the xar archive format, +// sufficient to produce macOS flat .pkg installers. It simulates macOS command +// `xar --compression none` invocation. +// +// A xar archive is: +// +// [ 28-byte header ][ zlib-compressed TOC (XML) ][ heap ] +// +// The header points at the compressed TOC; the TOC is an XML description of the +// file tree whose / elements reference byte ranges ("offset" and +// "length") within the heap. The heap begins with the SHA-1 checksum of the +// compressed TOC (as declared by the TOC's own element), followed by +// each file's contents. Files are stored uncompressed (encoding +// application/octet-stream), matching the previous `--compression none` behavior. +// +// Reference: the xar on-disk format: https://github.com/mackyle/xar. + +const ( + xarMagic uint32 = 0x78617221 // "xar!" + xarHeaderSize uint16 = 28 + xarVersion uint16 = 1 + xarChecksumSHA1 uint32 = 1 // cksum_alg value for SHA-1 + xarChecksumSize int64 = 20 // size of a SHA-1 digest in bytes +) + +// xarEntry is a node in the archive tree. +type xarEntry struct { + name string + isDir bool + mode os.FileMode + data []byte // file contents (nil for directories) + children []*xarEntry // populated for directories + + // Populated during heap layout (files only): + id int + offset int64 + size int64 + sha1 string +} + +// writeXar walks srcDir and writes an uncompressed xar archive of its contents +// to dstPath. The archive tree is rooted at srcDir's children (srcDir itself is +// not included as a node), mirroring `xar -cf dst -C srcDir `. +func writeXar(srcDir, dstPath string) error { + entries, err := buildXarTree(srcDir) + if err != nil { + return fmt.Errorf("build xar tree: %w", err) + } + + // Lay out the heap. Offset 0 is reserved for the compressed-TOC checksum, + // so file data starts at xarChecksumSize. + var heap bytes.Buffer + cursor := xarChecksumSize + nextID := 1 + if err := layoutXarHeap(entries, &heap, &cursor, &nextID); err != nil { + return err + } + + // Build and compress the TOC. + toc := buildXarTOC(entries) + var compressed bytes.Buffer + zw := zlib.NewWriter(&compressed) + if _, err := zw.Write(toc); err != nil { + return fmt.Errorf("compress toc: %w", err) + } + if err := zw.Close(); err != nil { + return fmt.Errorf("close toc writer: %w", err) + } + + tocChecksum := sha1.Sum(compressed.Bytes()) //nolint:gosec // required by the xar format + + // Assemble the archive: header + compressed TOC + heap(checksum + data). + var out bytes.Buffer + if err := writeXarHeader(&out, len(compressed.Bytes()), len(toc)); err != nil { + return err + } + out.Write(compressed.Bytes()) + out.Write(tocChecksum[:]) + out.Write(heap.Bytes()) + + if err := os.WriteFile(dstPath, out.Bytes(), 0o644); err != nil { + return fmt.Errorf("write xar: %w", err) + } + return nil +} + +// writeXarHeader writes the 28-byte big-endian xar header. +func writeXarHeader(w *bytes.Buffer, compressedTOCLen, uncompressedTOCLen int) error { + fields := []any{ + xarMagic, + xarHeaderSize, + xarVersion, + uint64(compressedTOCLen), //nolint:gosec // slice length is non-negative + uint64(uncompressedTOCLen), //nolint:gosec // slice length is non-negative + xarChecksumSHA1, + } + for _, f := range fields { + if err := binary.Write(w, binary.BigEndian, f); err != nil { + return fmt.Errorf("write xar header: %w", err) + } + } + return nil +} + +// buildXarTree reads dir and returns its immediate children as xar entries, +// recursing into subdirectories. Entries are sorted by name for deterministic +// output. +func buildXarTree(dir string) ([]*xarEntry, error) { + dirEntries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + + var entries []*xarEntry + for _, de := range dirEntries { + full := filepath.Join(dir, de.Name()) + info, err := de.Info() + if err != nil { + return nil, err + } + + entry := &xarEntry{name: de.Name(), mode: info.Mode().Perm()} + if de.IsDir() { + entry.isDir = true + children, err := buildXarTree(full) + if err != nil { + return nil, err + } + entry.children = children + } else { + data, err := os.ReadFile(full) + if err != nil { + return nil, err + } + entry.data = data + } + entries = append(entries, entry) + } + + sort.Slice(entries, func(i, j int) bool { return entries[i].name < entries[j].name }) + return entries, nil +} + +// layoutXarHeap walks the tree in depth-first order, appending each file's data +// to heap and recording its id, offset, size, and checksum. Directories consume +// no heap space but still receive an id. cursor tracks the next free heap offset. +func layoutXarHeap(entries []*xarEntry, heap *bytes.Buffer, cursor *int64, nextID *int) error { + for _, e := range entries { + e.id = *nextID + *nextID++ + + if e.isDir { + if err := layoutXarHeap(e.children, heap, cursor, nextID); err != nil { + return err + } + continue + } + + sum := sha1.Sum(e.data) //nolint:gosec // required by the xar format + e.sha1 = fmt.Sprintf("%x", sum) + e.size = int64(len(e.data)) + e.offset = *cursor + heap.Write(e.data) + *cursor += e.size + } + return nil +} + +// buildXarTOC renders the TOC XML for the archive tree. +func buildXarTOC(entries []*xarEntry) []byte { + var b strings.Builder + b.WriteString(`` + "\n") + b.WriteString("\n") + b.WriteString(" \n") + b.WriteString(` ` + "\n") + fmt.Fprintf(&b, " %d\n", xarChecksumSize) + b.WriteString(" 0\n") + b.WriteString(" \n") + for _, e := range entries { + writeXarTOCEntry(&b, e, 2) + } + b.WriteString(" \n") + b.WriteString("\n") + return []byte(b.String()) +} + +// writeXarTOCEntry renders a single element (recursing into directory +// children) at the given indentation depth. +func writeXarTOCEntry(b *strings.Builder, e *xarEntry, depth int) { + ind := strings.Repeat(" ", depth) + fmt.Fprintf(b, "%s\n", ind, e.id) + fmt.Fprintf(b, "%s %s\n", ind, xarEscape(e.name)) + if e.isDir { + fmt.Fprintf(b, "%s directory\n", ind) + } else { + fmt.Fprintf(b, "%s file\n", ind) + } + fmt.Fprintf(b, "%s 0%o\n", ind, e.mode) + fmt.Fprintf(b, "%s 0\n", ind) + fmt.Fprintf(b, "%s 80\n", ind) + + if e.isDir { + for _, c := range e.children { + writeXarTOCEntry(b, c, depth+1) + } + } else { + fmt.Fprintf(b, "%s \n", ind) + fmt.Fprintf(b, "%s %s\n", ind, e.sha1) + fmt.Fprintf(b, "%s %s\n", ind, e.sha1) + fmt.Fprintf(b, "%s %d\n", ind, e.size) + fmt.Fprintf(b, "%s %d\n", ind, e.offset) + fmt.Fprintf(b, "%s \n", ind) + fmt.Fprintf(b, "%s %d\n", ind, e.size) + fmt.Fprintf(b, "%s \n", ind) + } + fmt.Fprintf(b, "%s\n", ind) +} + +// xarEscape escapes the small set of characters that can appear in a file name +// and would otherwise be invalid in the TOC XML. +func xarEscape(s string) string { + r := strings.NewReplacer( + "&", "&", + "<", "<", + ">", ">", + `"`, """, + "'", "'", + ) + return r.Replace(s) +} diff --git a/orbit/pkg/packaging/xar_test.go b/orbit/pkg/packaging/xar_test.go new file mode 100644 index 0000000000..9237b12cb8 --- /dev/null +++ b/orbit/pkg/packaging/xar_test.go @@ -0,0 +1,130 @@ +package packaging + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/fleetdm/fleet/v4/pkg/file" + "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/stretchr/testify/require" +) + +// These tests exercise the pure-Go xar writer (writeXar) by feeding its output +// to Fleet's own xar decoder in pkg/file. A xar that round-trips through the +// decoder proves the encoder produces a well-formed header, a valid zlib TOC, +// and correct heap offsets/lengths (the decoder reads members back by those +// offsets). The writer is platform-independent, so these run everywhere. + +const testDistribution = ` + + Fleet osquery + + + +` + +const testPackageInfo = ` +` + +// writeXarTree writes the given name->contents map (names may contain "/" to +// create nested directories) into a fresh temp dir and returns its path. +func writeXarTree(t *testing.T, files map[string][]byte) string { + t.Helper() + root := t.TempDir() + for name, data := range files { + p := filepath.Join(root, name) + require.NoError(t, os.MkdirAll(filepath.Dir(p), 0o755)) + require.NoError(t, os.WriteFile(p, data, 0o644)) + } + return root +} + +// buildXar runs writeXar over root and returns the archive bytes. +func buildXar(t *testing.T, root string) []byte { + t.Helper() + out := filepath.Join(t.TempDir(), "out.pkg") + require.NoError(t, writeXar(root, out)) + b, err := os.ReadFile(out) + require.NoError(t, err) + return b +} + +func extractXARMetadata(t *testing.T, xarBytes []byte) *file.InstallerMetadata { + t.Helper() + tfr, err := fleet.NewTempFileReader(bytes.NewReader(xarBytes), t.TempDir) + require.NoError(t, err) + t.Cleanup(func() { _ = tfr.Close() }) + meta, err := file.ExtractXARMetadata(tfr) + require.NoError(t, err) + return meta +} + +// TestWriteXarReadableByDecoder builds a distribution-style .pkg tree and +// verifies the pure-Go writer's output is a valid xar: parseable header + TOC, +// a discoverable Distribution member, and metadata read back correctly from the +// heap. +func TestWriteXarReadableByDecoder(t *testing.T) { + root := writeXarTree(t, map[string][]byte{ + "Distribution": []byte(testDistribution), + "base.pkg/PackageInfo": []byte(testPackageInfo), + "base.pkg/Payload": bytes.Repeat([]byte("orbit-payload-bytes\n"), 1000), + }) + xarBytes := buildXar(t, root) + + // Valid, unsigned xar: exercises magic-byte check, SHA-1 hash-type mapping, + // zlib TOC decompression, and TOC XML parsing. + require.ErrorIs(t, file.CheckPKGSignature(bytes.NewReader(xarBytes)), file.ErrNotSigned) + + // The TOC lists the top-level Distribution file. + hasDist, err := file.XARHasDistribution(bytes.NewReader(xarBytes)) + require.NoError(t, err) + require.True(t, hasDist) + + // Reading the Distribution member back (via its / within the + // heap that begins with the 20-byte TOC checksum) yields exactly the bytes we + // wrote; if any offset/length were off the XML parse would fail. + meta := extractXARMetadata(t, xarBytes) + require.Equal(t, "Fleet osquery", meta.Name) + require.Equal(t, "1.2.3", meta.Version) + require.Equal(t, "com.fleetdm.orbit", meta.BundleIdentifier) + require.Contains(t, meta.PackageIDs, "com.fleetdm.orbit") +} + +// TestWriteXarPackageInfoFallback verifies a component-style .pkg (top-level +// PackageInfo, no Distribution) also round-trips: the decoder falls back to +// PackageInfo, which again requires the writer's heap offsets to be correct. +func TestWriteXarPackageInfoFallback(t *testing.T) { + root := writeXarTree(t, map[string][]byte{ + "PackageInfo": []byte(testPackageInfo), + }) + xarBytes := buildXar(t, root) + + hasDist, err := file.XARHasDistribution(bytes.NewReader(xarBytes)) + require.NoError(t, err) + require.False(t, hasDist) + + meta := extractXARMetadata(t, xarBytes) + require.Equal(t, "9.9.9", meta.Version) + require.Equal(t, "com.fleetdm.orbit", meta.BundleIdentifier) +} + +// TestWriteXarEmptyAndNestedDirs ensures the writer handles empty files and +// nested directories without corrupting the archive (the decoder still parses +// the header/TOC and finds the Distribution). +func TestWriteXarEmptyAndNestedDirs(t *testing.T) { + root := writeXarTree(t, map[string][]byte{ + "Distribution": []byte(testDistribution), + "base.pkg/empty": {}, + "base.pkg/nested/deep/Info.plist": []byte(""), + }) + // An empty directory too (map above only creates dirs with files). + require.NoError(t, os.MkdirAll(filepath.Join(root, "Resources"), 0o755)) + + xarBytes := buildXar(t, root) + require.ErrorIs(t, file.CheckPKGSignature(bytes.NewReader(xarBytes)), file.ErrNotSigned) + + meta := extractXARMetadata(t, xarBytes) + require.Equal(t, "Fleet osquery", meta.Name) +}