diff --git a/changes/18318-extract-metadata-from-installers b/changes/18318-extract-metadata-from-installers new file mode 100644 index 0000000000..c504760224 --- /dev/null +++ b/changes/18318-extract-metadata-from-installers @@ -0,0 +1 @@ +* Added support to extract package name and version from software installers. diff --git a/go.mod b/go.mod index 87e3ac429b..4aa773996d 100644 --- a/go.mod +++ b/go.mod @@ -202,6 +202,7 @@ require ( github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/elastic/go-sysinfo v1.7.1 // indirect github.com/elastic/go-windows v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect @@ -275,6 +276,7 @@ require ( github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/saferwall/pe v1.5.2 // indirect github.com/secure-systems-lab/go-securesystemslib v0.5.0 // indirect github.com/sergi/go-diff v1.2.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect @@ -295,6 +297,7 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yashtewari/glob-intersection v0.1.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect diff --git a/go.sum b/go.sum index b42024dcc0..8dd5d62aee 100644 --- a/go.sum +++ b/go.sum @@ -407,6 +407,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/e-dard/netbug v0.0.0-20151029172837-e64d308a0b20 h1:eDPsdileewX4H5a2Jph4gS8mFf749gzIrzpbnPy1oRs= github.com/e-dard/netbug v0.0.0-20151029172837-e64d308a0b20/go.mod h1:WXFUXJ0Y/SzNqXmhUU7VkE7a2Pag0zZnE2b6I87YWIs= +github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= +github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/elastic/go-licenser v0.4.0/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU= github.com/elastic/go-sysinfo v1.7.1 h1:Wx4DSARcKLllpKT2TnFVdSUJOsybqMYCNQZq1/wO+s0= github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= @@ -1062,6 +1064,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/saferwall/pe v1.5.2 h1:h5lLtLsyxGHQ9dN6cd8EfeLEBEo5gdqJpkuw4o4vTMY= +github.com/saferwall/pe v1.5.2/go.mod h1:SNzv3cdgk8SBI0UwHfyTcdjawfdnN+nbydnEL7GZ25s= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/scjalliance/comshim v0.0.0-20230315213746-5e51f40bd3b9 h1:rc/CcqLH3lh8n+csdOuDfP+NuykE0U6AeYSJJHKDgSg= github.com/scjalliance/comshim v0.0.0-20230315213746-5e51f40bd3b9/go.mod h1:a/83NAfUXvEuLpmxDssAXxgUgrEy12MId3Wd7OTs76s= diff --git a/pkg/file/deb.go b/pkg/file/deb.go new file mode 100644 index 0000000000..07f51cf6c7 --- /dev/null +++ b/pkg/file/deb.go @@ -0,0 +1,134 @@ +package file + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/bzip2" + "compress/gzip" + "errors" + "fmt" + "io" + "path" + "path/filepath" + "strings" + + "github.com/blakesmith/ar" + "github.com/xi2/xz" +) + +// ExtractDebMetadata extracts the name and version metadata from a .deb file , +// a debian installer package which is in archive format. +func ExtractDebMetadata(b []byte) (name, version string, err error) { + r := ar.NewReader(bytes.NewReader(b)) + + for { + hdr, err := r.Next() + if err == io.EOF { + break + } else if err != nil { + return "", "", fmt.Errorf("failed to advance to next file in archive: %w", err) + } + + name := path.Clean(hdr.Name) + if strings.HasPrefix(name, "control.tar") { + ext := filepath.Ext(name) + if ext == ".tar" { + ext = "" + } + return parseControl(r, ext) + } + } + + // no control.tar file found, return empty information + return "", "", nil +} + +// parseControl adapted from +// https://github.com/sassoftware/relic/blob/6c510a666832163a5d02587bda8be970d5e29b8c/lib/signdeb/control.go#L38-L39 +// +// Copyright (c) SAS Institute Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Parse basic package info from a control.tar.* stream. +func parseControl(r io.Reader, ext string) (name, version string, err error) { + switch ext { + case ".gz": + gz, err := gzip.NewReader(r) + if err != nil { + return "", "", fmt.Errorf("failed to create gzip reader: %w", err) + } + defer gz.Close() + r = gz + + case ".bz2": + r = bzip2.NewReader(r) + case ".xz": + r, err = xz.NewReader(r, 0) + if err != nil { + return "", "", fmt.Errorf("failed to create xz reader: %w", err) + } + case "": + // uncompressed + default: + return "", "", errors.New("unrecognized compression on control.tar: " + ext) + } + + tr := tar.NewReader(r) + found := false + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + return "", "", err + } + if path.Clean(hdr.Name) == "control" { + found = true + break + } + } + + if !found { + return "", "", errors.New("control.tar has no control file") + } + + blob, err := io.ReadAll(tr) + if err != nil { + return "", "", fmt.Errorf("failed to read tar file: %w", err) + } + + scanner := bufio.NewScanner(bytes.NewReader(blob)) + for scanner.Scan() { + line := scanner.Text() + i := strings.IndexAny(line, " \t\r\n") + j := strings.Index(line, ":") + if j < 0 || i < j { + continue + } + + key := line[:j] + value := strings.Trim(line[j+1:], " \t\r\n") + switch strings.ToLower(key) { + case "package": + name = value + case "version": + version = value + } + } + if err := scanner.Err(); err != nil { + return name, version, fmt.Errorf("failed to scan control file: %w", err) + } + return name, version, nil +} diff --git a/pkg/file/file.go b/pkg/file/file.go index c40d6390b4..23745468bb 100644 --- a/pkg/file/file.go +++ b/pkg/file/file.go @@ -11,6 +11,22 @@ import ( "github.com/fleetdm/fleet/v4/pkg/secure" ) +// ExtractInstallerMetadata extracts the software name and version from the +// installer file. The format of the installer is determined based on the +// extension of the filename. +func ExtractInstallerMetadata(filename string, b []byte) (name, version string, err error) { + switch ext := filepath.Ext(filename); ext { + case ".deb": + return ExtractDebMetadata(b) + case ".exe": + return ExtractPEMetadata(b) + case ".pkg": + return ExtractXARMetadata(b) + default: + return "", "", fmt.Errorf("unsupported file type: %s", ext) + } +} + // Copy copies the file from srcPath to dstPath, using the provided permissions. // // Note that on Windows the permissions support is limited in Go's file functions. diff --git a/pkg/file/file_test.go b/pkg/file/file_test.go index f40a84f698..3e0bd9c7b7 100644 --- a/pkg/file/file_test.go +++ b/pkg/file/file_test.go @@ -4,6 +4,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" "testing" "github.com/fleetdm/fleet/v4/pkg/file" @@ -90,3 +91,46 @@ func TestExists(t *testing.T) { require.NoError(t, err) assert.False(t, exists) } + +// TestExtractInstallerMetadata tests the ExtractInstallerMetadata function. It +// calls the function for every file under testdata/installers and checks that +// it returns the expected metadata by comparing it to the software name and +// version in the filename. +// +// The filename should have the following format: +// +// $[$]. +// +// That is, it breaks the file name at the dollar sign and the first part is +// the expected name, the second is the expected version. Note that by default, +// files in testdata/installers are NOT included in git, so the test files must +// be added manually (for size and licenses considerations). Why the dollar +// sign? Because dots, dashes and underlines are more likely to be part of the +// name or version. +func TestExtractInstallerMetadata(t *testing.T) { + dents, err := os.ReadDir(filepath.Join("testdata", "installers")) + if err != nil { + t.Fatal(err) + } + + for _, dent := range dents { + if !dent.Type().IsRegular() || strings.HasPrefix(dent.Name(), ".") { + continue + } + t.Run(dent.Name(), func(t *testing.T) { + parts := strings.Split(strings.TrimSuffix(dent.Name(), filepath.Ext(dent.Name())), "$") + if len(parts) < 2 { + t.Fatalf("invalid filename, expected at least 2 sections, got %d: %s", len(parts), dent.Name()) + } + wantName, wantVersion := parts[0], parts[1] + + content, err := os.ReadFile(filepath.Join("testdata", "installers", dent.Name())) + require.NoError(t, err) + + name, version, err := file.ExtractInstallerMetadata(dent.Name(), content) + require.NoError(t, err) + assert.Equal(t, wantName, name) + assert.Equal(t, wantVersion, version) + }) + } +} diff --git a/pkg/file/pe.go b/pkg/file/pe.go new file mode 100644 index 0000000000..3f6220d264 --- /dev/null +++ b/pkg/file/pe.go @@ -0,0 +1,45 @@ +package file + +import ( + "fmt" + "strings" + + "github.com/saferwall/pe" +) + +// ExtractPEMetadata extracts the name and version metadata from a .exe file in +// the Portable Executable (PE) format. +func ExtractPEMetadata(b []byte) (name, version string, err error) { + // cannot use the "Fast" option, we need the data directories for the + // resources to be available. + pep, err := pe.NewBytes(b, &pe.Options{ + OmitExportDirectory: true, + OmitImportDirectory: true, + OmitExceptionDirectory: true, + OmitSecurityDirectory: true, + OmitRelocDirectory: true, + OmitDebugDirectory: true, + OmitArchitectureDirectory: true, + OmitGlobalPtrDirectory: true, + OmitTLSDirectory: true, + OmitLoadConfigDirectory: true, + OmitBoundImportDirectory: true, + OmitIATDirectory: true, + OmitDelayImportDirectory: true, + OmitCLRHeaderDirectory: true, + }) + if err != nil { + return "", "", fmt.Errorf("error creating PE file: %w", err) + } + defer pep.Close() + + if err := pep.Parse(); err != nil { + return "", "", fmt.Errorf("error parsing PE file: %w", err) + } + + v, err := pep.ParseVersionResources() + if err != nil { + return "", "", fmt.Errorf("error parsing PE version resources: %w", err) + } + return strings.TrimSpace(v["ProductName"]), strings.TrimSpace(v["ProductVersion"]), nil +} diff --git a/pkg/file/testdata/installers/.gitignore b/pkg/file/testdata/installers/.gitignore new file mode 100644 index 0000000000..206053cb6e --- /dev/null +++ b/pkg/file/testdata/installers/.gitignore @@ -0,0 +1,6 @@ +# ignore everything except gitignore +# software installers can be added locally to test the ExtractInstallerMetadata +# logic, but tend to be big binary files with various licenses that might not +# make it possible to include in the repository. +* +!.gitignore diff --git a/pkg/file/xar.go b/pkg/file/xar.go index 71566c0687..0bbe49d278 100644 --- a/pkg/file/xar.go +++ b/pkg/file/xar.go @@ -27,12 +27,17 @@ import ( "errors" "fmt" "io" + "strings" ) -// xarMagic is the [file signature][1] (or magic bytes) for xar -// -// [1]: https://en.wikipedia.org/wiki/List_of_file_signatures -const xarMagic = 0x78617221 +const ( + // xarMagic is the [file signature][1] (or magic bytes) for xar + // + // [1]: https://en.wikipedia.org/wiki/List_of_file_signatures + xarMagic = 0x78617221 + + xarHeaderSize = 28 +) const ( hashNone uint32 = iota @@ -69,6 +74,108 @@ type toc struct { XSignature *any `xml:"x-signature"` } +type xmlXar struct { + XMLName xml.Name `xml:"xar"` + TOC xmlTOC +} + +type xmlTOC struct { + XMLName xml.Name `xml:"toc"` + Files []*xmlFile `xml:"file"` +} + +type xmlFileData struct { + XMLName xml.Name `xml:"data"` + Length int64 `xml:"length"` + Offset int64 `xml:"offset"` + Size int64 `xml:"size"` + Encoding struct { + Style string `xml:"style,attr"` + } `xml:"encoding"` +} + +type xmlFile struct { + XMLName xml.Name `xml:"file"` + Name string `xml:"name"` + Data *xmlFileData +} + +type distributionXML struct { + PkgRef []pkgRef `xml:"pkg-ref"` +} + +type pkgRef struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr,omitempty"` + Auth string `xml:"auth,attr,omitempty"` + Content string `xml:",chardata"` +} + +// ExtractXARMetadata extracts the name and version metadata from a .pkg file +// in the XAR format. +func ExtractXARMetadata(b []byte) (name, version string, err error) { + var hdr xarHeader + + r := bytes.NewReader(b) + if err := binary.Read(r, binary.BigEndian, &hdr); err != nil { + return "", "", fmt.Errorf("decode xar header: %w", err) + } + + zr, err := zlib.NewReader(io.LimitReader(r, hdr.CompressedSize)) + if err != nil { + return "", "", fmt.Errorf("create zlib reader: %w", err) + } + defer zr.Close() + + var root xmlXar + decoder := xml.NewDecoder(zr) + decoder.Strict = false + if err := decoder.Decode(&root); err != nil { + return "", "", fmt.Errorf("decode xar xml: %w", err) + } + + heapOffset := xarHeaderSize + hdr.CompressedSize + for _, f := range root.TOC.Files { + if f.Name == "Distribution" { + var fileReader io.Reader + heapReader := io.NewSectionReader(r, heapOffset, int64(len(b))-heapOffset) + fileReader = io.NewSectionReader(heapReader, f.Data.Offset, f.Data.Length) + + // the distribution file can be compressed differently than the TOC, the + // actual compression is specified in the Encoding.Style field. + if strings.Contains(f.Data.Encoding.Style, "x-gzip") { + // despite the name, x-gzip fails to decode with the gzip package + // (invalid header), but it works with zlib. + zr, err := zlib.NewReader(fileReader) + if err != nil { + return "", "", fmt.Errorf("create zlib reader: %w", err) + } + defer zr.Close() + fileReader = zr + + // TODO(mna): obviously, we may need to support more decompression methods here... + } + + contents, err := io.ReadAll(fileReader) + if err != nil { + return "", "", fmt.Errorf("reading Distribution file: %w", err) + } + + var distXML distributionXML + if err := xml.Unmarshal(contents, &distXML); err != nil { + return "", "", fmt.Errorf("unmarshal Distribution XML: %w", err) + } + + if len(distXML.PkgRef) > 0 { + return strings.TrimSpace(distXML.PkgRef[0].ID), strings.TrimSpace(distXML.PkgRef[0].Version), nil + } + break + } + } + + return "", "", nil +} + // CheckPKGSignature checks if the provided bytes correspond to a signed pkg // (xar) file. //