<!-- Add the related story/sub-task/bug number, like Resolves #123, or remove if NA --> **Related issue:** NA — resolves high-severity code-scanning alerts in `tools/dibble` ## Problem The dibble seeder committed 14 installer fixtures under `tools/dibble/pkg/seed/data/installers/` and bundled them into the binary with `//go:embed`. The `.exe`/`.msi`/`.deb`/`.rpm` files tripped **8 high-severity "Binary-Artifacts" code-scanning alerts** on `main`. 11 of the 14 are byte-identical to fixtures already in `server/service/testdata/software-installers/`, but `go:embed` can't reference files outside the package (no `..`, no symlinks), so the copies couldn't simply point at the originals. ## Change Replace the embed with an on-demand fetch that runs **only when seeding software** (`SoftwareCustom`): - Fixtures are downloaded, **SHA-256 verified** against a pinned manifest, and cached under the user cache dir (`os.UserCacheDir()/dibble/installers`) so repeat runs stay offline. - Fixtures shared with Fleet's tests are pulled from `testdata` via `raw.githubusercontent.com` at a **pinned commit**; the 7-Zip and python-manager installers come from their **upstream URLs**. - No binaries remain committed in this module. Checksums for all sources were verified to match the previously-committed bytes exactly, and the download + verify + cache path was smoke-tested locally. ## Tradeoff `dibble software custom` now requires network access on first use (downloads are cached afterward). This only affects the software-seeding path; all other seeders are unchanged. # Checklist for submitter - [x] Input data is properly validated (downloaded fixtures are rejected unless their SHA-256 matches the pinned manifest), `SELECT *` is avoided, SQL injection is prevented, JS inline code is prevented, and untrusted data interpolated into shell scripts/commands is validated against shell metacharacters. - [x] Timeouts are implemented and retries are limited to avoid infinite loops (HTTP client uses a 2-minute timeout; no retry loop). ## Testing - [x] QA'd all new/changed functionality manually (verified download, checksum verification, and cache reuse for testdata- and upstream-hosted fixtures) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Installer fixtures are now downloaded on demand and cached locally for faster repeat runs. * Expanded installer fixture coverage, including additional Windows-signed installers for improved platform support. * **Bug Fixes** * Added SHA-256 integrity verification for cached and newly downloaded installer fixtures. * Improved reliability and safety by re-downloading when cache contents don’t match and by writing downloads atomically to avoid partial files. * **Chores** * Updated indirect dependency versions related to OpenTelemetry and `golang.org/x/*`. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
210 lines
7.0 KiB
Go
210 lines
7.0 KiB
Go
package seed
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
// extensionInstallers lists the curated 2-3 installer fixtures per
|
|
// extension. Order matters for display; the first entry per extension is
|
|
// uploaded first which keeps log output readable. fleet-osquery.msi is
|
|
// intentionally NOT listed: it's the agent installer and must not appear
|
|
// as a custom software item.
|
|
var extensionInstallers = map[string][]string{
|
|
".pkg": {"dummy_installer.pkg", "EchoApp.pkg", "no_version.pkg"},
|
|
".deb": {"emacs.deb", "ruby.deb", "ruby_arm64.deb"},
|
|
".msi": {"python-manager-26.2.msi"},
|
|
".exe": {"7z2601.exe", "7z2601-x64.exe", "7z2601-arm64.exe"},
|
|
".rpm": {"ruby.rpm"}, // only fixture available
|
|
".tar.gz": {"test.tar.gz"}, // only fixture available
|
|
".ipa": {"ipa_test.ipa", "ipa_test2.ipa"},
|
|
}
|
|
|
|
// extensionScripts maps an extension to the install / uninstall script form
|
|
// field values to send with the upload. Most extensions are left empty so
|
|
// Fleet auto-generates the commands. .tar.gz and .exe both reject uploads
|
|
// without explicit install scripts ("Install script is required for .X
|
|
// packages") so we ship placeholders that satisfy the validator.
|
|
var extensionScripts = map[string]struct {
|
|
install string
|
|
uninstall string
|
|
}{
|
|
".tar.gz": {
|
|
install: "#!/bin/sh\necho 'dibble seeded — replace with real install logic'\n",
|
|
uninstall: "#!/bin/sh\necho 'dibble seeded — replace with real uninstall logic'\n",
|
|
},
|
|
".exe": {
|
|
install: "# dibble seeded — replace with real install logic\n$exitCode = (Start-Process -FilePath $env:INSTALLER_PATH -ArgumentList \"/S\" -PassThru -Wait).ExitCode\nExit $exitCode\n",
|
|
uninstall: "# dibble seeded — replace with real uninstall logic\nExit 0\n",
|
|
},
|
|
}
|
|
|
|
// SoftwareOptions configures the custom-package and Fleet-maintained-app
|
|
// seeders. TeamID == 0 targets "no team" (global); a non-zero value scopes
|
|
// the upload to that team.
|
|
type SoftwareOptions struct {
|
|
// TeamID selects the team that uploaded installers and added maintained
|
|
// apps land under. Zero means no team / global.
|
|
TeamID uint
|
|
|
|
// MaintainedAppCount is how many entries from /software/fleet_maintained_apps
|
|
// to seed. Zero skips FMA entirely. Defaults to 3 when running the
|
|
// "maintained" / "all" subcommands.
|
|
MaintainedAppCount int
|
|
}
|
|
|
|
// sortedExtensions returns the supported extensions in a deterministic
|
|
// order so seeded output is stable across runs.
|
|
func sortedExtensions() []string {
|
|
keys := make([]string, 0, len(extensionInstallers))
|
|
for k := range extensionInstallers {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
return keys
|
|
}
|
|
|
|
// SoftwareCustom uploads the curated 2-3 installer fixtures per supported
|
|
// extension to the given team (or "no team" when opt.TeamID == 0). Each
|
|
// upload posts multipart to /api/latest/fleet/software/package, the same
|
|
// endpoint Fleet's UI calls when adding a custom package.
|
|
//
|
|
// Install / uninstall scripts are left empty; the server auto-generates
|
|
// defaults based on the file extension.
|
|
func SoftwareCustom(c Client, log Logger, opt SoftwareOptions) Result {
|
|
res := Result{Entity: "software (custom)"}
|
|
|
|
scope := "no team"
|
|
teamField := ""
|
|
if opt.TeamID > 0 {
|
|
teamField = fmt.Sprintf("%d", opt.TeamID)
|
|
scope = fmt.Sprintf("team=%d", opt.TeamID)
|
|
}
|
|
|
|
for _, ext := range sortedExtensions() {
|
|
for _, fixture := range extensionInstallers[ext] {
|
|
content, err := loadInstaller(log, fixture)
|
|
if err != nil {
|
|
res.Errors = append(res.Errors,
|
|
fmt.Errorf("load %s: %w", fixture, err))
|
|
continue
|
|
}
|
|
// Build the fields map per-fixture: extensions like .tar.gz
|
|
// need an explicit install_script, others let Fleet
|
|
// auto-generate one.
|
|
fields := map[string]string{}
|
|
if teamField != "" {
|
|
fields["fleet_id"] = teamField
|
|
}
|
|
if scripts, ok := extensionScripts[ext]; ok {
|
|
fields["install_script"] = scripts.install
|
|
fields["uninstall_script"] = scripts.uninstall
|
|
}
|
|
files := []MultipartFile{{
|
|
FieldName: "software",
|
|
Filename: fixture,
|
|
Content: content,
|
|
}}
|
|
err = c.PostMultipart("/api/latest/fleet/software/package", fields, files, nil)
|
|
switch {
|
|
case err == nil:
|
|
res.Created++
|
|
log.Printf("software (%s) %s [%s]", scope, fixture, ext)
|
|
case IsAlreadyExists(err):
|
|
res.Skipped++
|
|
log.Printf("software (%s) %s already exists", scope, fixture)
|
|
default:
|
|
res.Errors = append(res.Errors,
|
|
fmt.Errorf("%s: %w", fixture, err))
|
|
}
|
|
}
|
|
}
|
|
return res
|
|
}
|
|
|
|
// maintainedApp is the subset of fleet.MaintainedApp the seeder cares
|
|
// about. Decoded from the listFleetMaintainedApps response.
|
|
type maintainedApp struct {
|
|
ID uint `json:"id"`
|
|
Name string `json:"name"`
|
|
Platform string `json:"platform"`
|
|
}
|
|
|
|
type listMaintainedAppsResponse struct {
|
|
FleetMaintainedApps []maintainedApp `json:"fleet_maintained_apps"`
|
|
}
|
|
|
|
// SoftwareMaintained adds a handful of Fleet-maintained apps to the given
|
|
// team by:
|
|
//
|
|
// 1. GET /api/latest/fleet/software/fleet_maintained_apps to discover what
|
|
// the server's catalog contains (the list is generated server-side).
|
|
// 2. POST /api/latest/fleet/software/fleet_maintained_apps with each
|
|
// selected fleet_maintained_app_id.
|
|
//
|
|
// Adding FMAs is per-team; opt.TeamID == 0 means no team / global.
|
|
func SoftwareMaintained(c Client, log Logger, opt SoftwareOptions) Result {
|
|
res := Result{Entity: "software (maintained)"}
|
|
if opt.MaintainedAppCount <= 0 {
|
|
return res
|
|
}
|
|
|
|
listPath := "/api/latest/fleet/software/fleet_maintained_apps"
|
|
if opt.TeamID > 0 {
|
|
listPath = fmt.Sprintf("%s?team_id=%d", listPath, opt.TeamID)
|
|
}
|
|
var list listMaintainedAppsResponse
|
|
if err := c.Get(listPath, &list); err != nil {
|
|
res.Errors = append(res.Errors,
|
|
fmt.Errorf("list fleet-maintained apps: %w", err))
|
|
return res
|
|
}
|
|
if len(list.FleetMaintainedApps) == 0 {
|
|
log.Printf("software (maintained): server returned no maintained apps to add")
|
|
return res
|
|
}
|
|
|
|
// Pick the first N from the server's list — the catalog is curated so
|
|
// the head of the list is stable.
|
|
n := opt.MaintainedAppCount
|
|
if n > len(list.FleetMaintainedApps) {
|
|
n = len(list.FleetMaintainedApps)
|
|
}
|
|
|
|
for i := 0; i < n; i++ {
|
|
app := list.FleetMaintainedApps[i]
|
|
body := map[string]any{
|
|
"fleet_maintained_app_id": app.ID,
|
|
}
|
|
if opt.TeamID > 0 {
|
|
body["fleet_id"] = opt.TeamID
|
|
}
|
|
err := c.Post("/api/latest/fleet/software/fleet_maintained_apps", body, nil)
|
|
switch {
|
|
case err == nil:
|
|
res.Created++
|
|
log.Printf("software (maintained) %s [%s] id=%d",
|
|
app.Name, app.Platform, app.ID)
|
|
case IsAlreadyExists(err) || isAlreadyAdded(err):
|
|
res.Skipped++
|
|
default:
|
|
res.Errors = append(res.Errors,
|
|
fmt.Errorf("add maintained app %s (id=%d): %w", app.Name, app.ID, err))
|
|
}
|
|
}
|
|
return res
|
|
}
|
|
|
|
// isAlreadyAdded recognizes the "already added" error Fleet returns when a
|
|
// maintained app is re-added to the same team. The error isn't a generic
|
|
// "already exists" 409, so we have to sniff the message.
|
|
func isAlreadyAdded(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
msg := strings.ToLower(err.Error())
|
|
return strings.Contains(msg, "already") &&
|
|
(strings.Contains(msg, "added") || strings.Contains(msg, "associated"))
|
|
}
|