Files
George Karr 55c1783900 dibble: download seed installer fixtures at runtime instead of embedding (#48823)
<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** NA — resolves high-severity code-scanning alerts in
`tools/dibble`

## Problem

The dibble seeder committed 14 installer fixtures under
`tools/dibble/pkg/seed/data/installers/` and bundled them into the
binary with
`//go:embed`. The `.exe`/`.msi`/`.deb`/`.rpm` files tripped **8
high-severity
"Binary-Artifacts" code-scanning alerts** on `main`.

11 of the 14 are byte-identical to fixtures already in
`server/service/testdata/software-installers/`, but `go:embed` can't
reference
files outside the package (no `..`, no symlinks), so the copies couldn't
simply
point at the originals.

## Change

Replace the embed with an on-demand fetch that runs **only when seeding
software** (`SoftwareCustom`):

- Fixtures are downloaded, **SHA-256 verified** against a pinned
manifest, and
cached under the user cache dir (`os.UserCacheDir()/dibble/installers`)
so
  repeat runs stay offline.
- Fixtures shared with Fleet's tests are pulled from `testdata` via
  `raw.githubusercontent.com` at a **pinned commit**; the 7-Zip and
  python-manager installers come from their **upstream URLs**.
- No binaries remain committed in this module.

Checksums for all sources were verified to match the
previously-committed bytes
exactly, and the download + verify + cache path was smoke-tested
locally.

## Tradeoff

`dibble software custom` now requires network access on first use
(downloads are
cached afterward). This only affects the software-seeding path; all
other
seeders are unchanged.

# Checklist for submitter

- [x] Input data is properly validated (downloaded fixtures are rejected
unless their SHA-256 matches the pinned manifest), `SELECT *` is
avoided, SQL injection is prevented, JS inline code is prevented, and
untrusted data interpolated into shell scripts/commands is validated
against shell metacharacters.
- [x] Timeouts are implemented and retries are limited to avoid infinite
loops (HTTP client uses a 2-minute timeout; no retry loop).

## Testing

- [x] QA'd all new/changed functionality manually (verified download,
checksum verification, and cache reuse for testdata- and upstream-hosted
fixtures)


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Installer fixtures are now downloaded on demand and cached locally for
faster repeat runs.
* Expanded installer fixture coverage, including additional
Windows-signed installers for improved platform support.
* **Bug Fixes**
* Added SHA-256 integrity verification for cached and newly downloaded
installer fixtures.
* Improved reliability and safety by re-downloading when cache contents
don’t match and by writing downloads atomically to avoid partial files.
* **Chores**
* Updated indirect dependency versions related to OpenTelemetry and
`golang.org/x/*`.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
2026-07-09 17:33:25 -05:00

210 lines
7.0 KiB
Go

package seed
import (
"fmt"
"sort"
"strings"
)
// extensionInstallers lists the curated 2-3 installer fixtures per
// extension. Order matters for display; the first entry per extension is
// uploaded first which keeps log output readable. fleet-osquery.msi is
// intentionally NOT listed: it's the agent installer and must not appear
// as a custom software item.
var extensionInstallers = map[string][]string{
".pkg": {"dummy_installer.pkg", "EchoApp.pkg", "no_version.pkg"},
".deb": {"emacs.deb", "ruby.deb", "ruby_arm64.deb"},
".msi": {"python-manager-26.2.msi"},
".exe": {"7z2601.exe", "7z2601-x64.exe", "7z2601-arm64.exe"},
".rpm": {"ruby.rpm"}, // only fixture available
".tar.gz": {"test.tar.gz"}, // only fixture available
".ipa": {"ipa_test.ipa", "ipa_test2.ipa"},
}
// extensionScripts maps an extension to the install / uninstall script form
// field values to send with the upload. Most extensions are left empty so
// Fleet auto-generates the commands. .tar.gz and .exe both reject uploads
// without explicit install scripts ("Install script is required for .X
// packages") so we ship placeholders that satisfy the validator.
var extensionScripts = map[string]struct {
install string
uninstall string
}{
".tar.gz": {
install: "#!/bin/sh\necho 'dibble seeded — replace with real install logic'\n",
uninstall: "#!/bin/sh\necho 'dibble seeded — replace with real uninstall logic'\n",
},
".exe": {
install: "# dibble seeded — replace with real install logic\n$exitCode = (Start-Process -FilePath $env:INSTALLER_PATH -ArgumentList \"/S\" -PassThru -Wait).ExitCode\nExit $exitCode\n",
uninstall: "# dibble seeded — replace with real uninstall logic\nExit 0\n",
},
}
// SoftwareOptions configures the custom-package and Fleet-maintained-app
// seeders. TeamID == 0 targets "no team" (global); a non-zero value scopes
// the upload to that team.
type SoftwareOptions struct {
// TeamID selects the team that uploaded installers and added maintained
// apps land under. Zero means no team / global.
TeamID uint
// MaintainedAppCount is how many entries from /software/fleet_maintained_apps
// to seed. Zero skips FMA entirely. Defaults to 3 when running the
// "maintained" / "all" subcommands.
MaintainedAppCount int
}
// sortedExtensions returns the supported extensions in a deterministic
// order so seeded output is stable across runs.
func sortedExtensions() []string {
keys := make([]string, 0, len(extensionInstallers))
for k := range extensionInstallers {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
// SoftwareCustom uploads the curated 2-3 installer fixtures per supported
// extension to the given team (or "no team" when opt.TeamID == 0). Each
// upload posts multipart to /api/latest/fleet/software/package, the same
// endpoint Fleet's UI calls when adding a custom package.
//
// Install / uninstall scripts are left empty; the server auto-generates
// defaults based on the file extension.
func SoftwareCustom(c Client, log Logger, opt SoftwareOptions) Result {
res := Result{Entity: "software (custom)"}
scope := "no team"
teamField := ""
if opt.TeamID > 0 {
teamField = fmt.Sprintf("%d", opt.TeamID)
scope = fmt.Sprintf("team=%d", opt.TeamID)
}
for _, ext := range sortedExtensions() {
for _, fixture := range extensionInstallers[ext] {
content, err := loadInstaller(log, fixture)
if err != nil {
res.Errors = append(res.Errors,
fmt.Errorf("load %s: %w", fixture, err))
continue
}
// Build the fields map per-fixture: extensions like .tar.gz
// need an explicit install_script, others let Fleet
// auto-generate one.
fields := map[string]string{}
if teamField != "" {
fields["fleet_id"] = teamField
}
if scripts, ok := extensionScripts[ext]; ok {
fields["install_script"] = scripts.install
fields["uninstall_script"] = scripts.uninstall
}
files := []MultipartFile{{
FieldName: "software",
Filename: fixture,
Content: content,
}}
err = c.PostMultipart("/api/latest/fleet/software/package", fields, files, nil)
switch {
case err == nil:
res.Created++
log.Printf("software (%s) %s [%s]", scope, fixture, ext)
case IsAlreadyExists(err):
res.Skipped++
log.Printf("software (%s) %s already exists", scope, fixture)
default:
res.Errors = append(res.Errors,
fmt.Errorf("%s: %w", fixture, err))
}
}
}
return res
}
// maintainedApp is the subset of fleet.MaintainedApp the seeder cares
// about. Decoded from the listFleetMaintainedApps response.
type maintainedApp struct {
ID uint `json:"id"`
Name string `json:"name"`
Platform string `json:"platform"`
}
type listMaintainedAppsResponse struct {
FleetMaintainedApps []maintainedApp `json:"fleet_maintained_apps"`
}
// SoftwareMaintained adds a handful of Fleet-maintained apps to the given
// team by:
//
// 1. GET /api/latest/fleet/software/fleet_maintained_apps to discover what
// the server's catalog contains (the list is generated server-side).
// 2. POST /api/latest/fleet/software/fleet_maintained_apps with each
// selected fleet_maintained_app_id.
//
// Adding FMAs is per-team; opt.TeamID == 0 means no team / global.
func SoftwareMaintained(c Client, log Logger, opt SoftwareOptions) Result {
res := Result{Entity: "software (maintained)"}
if opt.MaintainedAppCount <= 0 {
return res
}
listPath := "/api/latest/fleet/software/fleet_maintained_apps"
if opt.TeamID > 0 {
listPath = fmt.Sprintf("%s?team_id=%d", listPath, opt.TeamID)
}
var list listMaintainedAppsResponse
if err := c.Get(listPath, &list); err != nil {
res.Errors = append(res.Errors,
fmt.Errorf("list fleet-maintained apps: %w", err))
return res
}
if len(list.FleetMaintainedApps) == 0 {
log.Printf("software (maintained): server returned no maintained apps to add")
return res
}
// Pick the first N from the server's list — the catalog is curated so
// the head of the list is stable.
n := opt.MaintainedAppCount
if n > len(list.FleetMaintainedApps) {
n = len(list.FleetMaintainedApps)
}
for i := 0; i < n; i++ {
app := list.FleetMaintainedApps[i]
body := map[string]any{
"fleet_maintained_app_id": app.ID,
}
if opt.TeamID > 0 {
body["fleet_id"] = opt.TeamID
}
err := c.Post("/api/latest/fleet/software/fleet_maintained_apps", body, nil)
switch {
case err == nil:
res.Created++
log.Printf("software (maintained) %s [%s] id=%d",
app.Name, app.Platform, app.ID)
case IsAlreadyExists(err) || isAlreadyAdded(err):
res.Skipped++
default:
res.Errors = append(res.Errors,
fmt.Errorf("add maintained app %s (id=%d): %w", app.Name, app.ID, err))
}
}
return res
}
// isAlreadyAdded recognizes the "already added" error Fleet returns when a
// maintained app is re-added to the same team. The error isn't a generic
// "already exists" 409, so we have to sniff the message.
func isAlreadyAdded(err error) bool {
if err == nil {
return false
}
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "already") &&
(strings.Contains(msg, "added") || strings.Contains(msg, "associated"))
}