<!-- Add the related story/sub-task/bug number, like Resolves #123, or remove if NA --> **Related issue:** Resolves #45728 Changes: - Adds a new redis key to keep track of downloaded packages. It starts out with an empty list and gets filled with each download. Each update writes the entire struct at once to the key. - Adds logging in the fleetctl gitops client to show which packages were downloaded - Fixes the categories key potentially expiring # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files) for more information. - [ ] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements), JS inline code is prevented especially for url redirects, and untrusted data interpolated into shell scripts/commands is validated against shell metacharacters. - ❌ Timeouts are implemented and retries are limited to avoid infinite loops - Right now the batch will write the whole slice of all packages to a single redis key for every package in the loop. Looks like performance is acceptable for now (500 packages), but maybe this will need to be limited. - [ ] If paths of existing endpoints are modified without backwards compatibility, checked the frontend/CLI for any necessary changes ## Testing - [x] Added/updated automated tests - [ ] Where appropriate, [automated tests simulate multiple hosts and test for host isolation](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/reference/patterns-backend.md#unit-testing) (updates to one hosts's records do not affect another) - [x] QA'd all new/changed functionality manually <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit ## New Features - Added per-package software download progress in fleetctl GitOps. - Progress now reports downloading, completed, skipped, and failed packages during real and dry runs. - Installation output now distinguishes applying and applied stages. ## Bug Fixes - Improved download error messages and cached-package handling. - Prevented duplicate progress messages and ensured tracking issues do not interrupt successful software batches. ## Tests - Expanded coverage for progress reporting, failures, dry runs, package types, and authorization scenarios. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
384 lines
13 KiB
Go
384 lines
13 KiB
Go
package service
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"net/url"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/fleetdm/fleet/v4/server/fleet"
|
|
)
|
|
|
|
// ListSoftwareVersions retrieves the software versions installed on hosts.
|
|
func (c *Client) ListSoftwareVersions(query string) ([]fleet.Software, error) {
|
|
verb, path := "GET", "/api/latest/fleet/software/versions"
|
|
var responseBody listSoftwareVersionsResponse
|
|
err := c.authenticatedRequestWithQuery(nil, verb, path, &responseBody, query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return responseBody.Software, nil
|
|
}
|
|
|
|
// ListSoftwareTitles retrieves the software titles installed on hosts.
|
|
func (c *Client) ListSoftwareTitles(query string) ([]fleet.SoftwareTitleListResult, error) {
|
|
verb, path := "GET", "/api/latest/fleet/software/titles"
|
|
var responseBody listSoftwareTitlesResponse
|
|
err := c.authenticatedRequestWithQuery(nil, verb, path, &responseBody, query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return responseBody.SoftwareTitles, nil
|
|
}
|
|
|
|
// Get the software titles available for the setup experience.
|
|
func (c *Client) GetSetupExperienceSoftware(platform string, teamID uint) ([]fleet.SoftwareTitleListResult, error) {
|
|
verb, path := "GET", "/api/latest/fleet/setup_experience/software"
|
|
var responseBody getSetupExperienceSoftwareResponse
|
|
query := fmt.Sprintf("platform=%s&fleet_id=%d", platform, teamID)
|
|
err := c.authenticatedRequestWithQuery(nil, verb, path, &responseBody, query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return responseBody.SoftwareTitles, nil
|
|
}
|
|
|
|
// GetSoftwareTitleByID retrieves a software title by ID.
|
|
//
|
|
//nolint:gocritic // ignore captLocal
|
|
func (c *Client) GetSoftwareTitleByID(ID uint, teamID *uint) (*fleet.SoftwareTitle, error) {
|
|
var query string
|
|
if teamID != nil {
|
|
query = fmt.Sprintf("fleet_id=%d", *teamID)
|
|
}
|
|
verb, path := "GET", "/api/latest/fleet/software/titles/"+fmt.Sprint(ID)
|
|
var responseBody getSoftwareTitleResponse
|
|
err := c.authenticatedRequestWithQuery(nil, verb, path, &responseBody, query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return responseBody.SoftwareTitle, nil
|
|
}
|
|
|
|
func (c *Client) GetSoftwareTitleIcon(titleID uint, teamID uint) ([]byte, error) {
|
|
verb, path := "GET", fmt.Sprintf("/api/latest/fleet/software/titles/%d/icon", titleID)
|
|
response, err := c.AuthenticatedDo(verb, path, fmt.Sprintf("fleet_id=%d", teamID), nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s %s: %w", verb, path, err)
|
|
}
|
|
defer response.Body.Close()
|
|
err = c.ParseResponse(verb, path, response, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parsing icon response: %w", err)
|
|
}
|
|
if response.StatusCode != http.StatusNoContent {
|
|
b, err := io.ReadAll(response.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("reading response body: %w", err)
|
|
}
|
|
return b, nil
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func (c *Client) ApplyNoTeamSoftwareInstallers(
|
|
softwareInstallers []fleet.SoftwareInstallerPayload,
|
|
opts fleet.ApplySpecOptions,
|
|
logFn func(format string, args ...any),
|
|
) ([]fleet.SoftwarePackageResponse, []fleet.DeletedSoftwarePackage, []string, error) {
|
|
query, err := url.ParseQuery(opts.RawQuery())
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
return c.applySoftwareInstallers(softwareInstallers, query, opts.DryRun, logFn)
|
|
}
|
|
|
|
func (c *Client) applySoftwareInstallers(
|
|
softwareInstallers []fleet.SoftwareInstallerPayload,
|
|
query url.Values,
|
|
dryRun bool,
|
|
logFn func(format string, args ...any),
|
|
) ([]fleet.SoftwarePackageResponse, []fleet.DeletedSoftwarePackage, []string, error) {
|
|
path := "/api/latest/fleet/software/batch"
|
|
var resp batchSetSoftwareInstallersResponse
|
|
if err := c.authenticatedRequestWithQuery(map[string]any{"software": softwareInstallers}, "POST", path, &resp, query.Encode()); err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
if dryRun && resp.RequestUUID == "" {
|
|
return nil, nil, nil, nil
|
|
}
|
|
|
|
// Keyed by place in the batch, since two packages can share a name.
|
|
printedDownloading := make(map[int]struct{})
|
|
printedResult := make(map[int]struct{})
|
|
|
|
// Assumes the server downloads packages one by one, so each "downloading" line prints
|
|
// right before its own "downloaded" line. Concurrent downloads would break this.
|
|
logDownloadProgress := func(downloadProgress []fleet.SoftwarePackageDownloadProgress) {
|
|
for payloadIndex, packageProgress := range downloadProgress {
|
|
// A package the batch hasn't started downloading has no name yet.
|
|
if packageProgress.Name == "" {
|
|
continue
|
|
}
|
|
|
|
// A package Fleet doesn't download gets only this line, never a downloading one.
|
|
if packageProgress.Status == fleet.SoftwarePackageDownloadSkipped {
|
|
_, printedSkip := printedResult[payloadIndex]
|
|
if !printedSkip {
|
|
printedResult[payloadIndex] = struct{}{}
|
|
logFn("[+] skipped downloading the software package (already in storage) - %s\n", packageProgress.Name)
|
|
}
|
|
continue
|
|
}
|
|
|
|
// A package can still turn out to be skipped after this prints, when the download returns a 304.
|
|
_, printedStart := printedDownloading[payloadIndex]
|
|
if !printedStart {
|
|
printedDownloading[payloadIndex] = struct{}{}
|
|
logFn("[+] downloading software package - %s ...\n", packageProgress.Name)
|
|
}
|
|
|
|
_, printedFinish := printedResult[payloadIndex]
|
|
if printedFinish {
|
|
continue
|
|
}
|
|
switch packageProgress.Status {
|
|
case fleet.SoftwarePackageDownloadFailed:
|
|
printedResult[payloadIndex] = struct{}{}
|
|
logFn("Error: could not download software package %s\n", packageProgress.Name)
|
|
case fleet.SoftwarePackageDownloadFinished:
|
|
printedResult[payloadIndex] = struct{}{}
|
|
logFn("[+] downloaded software package - %s\n", packageProgress.Name)
|
|
}
|
|
}
|
|
}
|
|
|
|
requestUUID := resp.RequestUUID
|
|
for {
|
|
var resp batchSetSoftwareInstallersResultResponse
|
|
if err := c.authenticatedRequestWithQuery(nil, "GET", path+"/"+requestUUID, &resp, query.Encode()); err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
logDownloadProgress(resp.DownloadProgress)
|
|
|
|
switch {
|
|
case resp.Status == fleet.BatchSetSoftwareInstallersStatusProcessing:
|
|
time.Sleep(1 * time.Second)
|
|
case resp.Status == fleet.BatchSetSoftwareInstallersStatusFailed:
|
|
return nil, nil, nil, errors.New(resp.Message)
|
|
case resp.Status == fleet.BatchSetSoftwareInstallersStatusCompleted:
|
|
return matchPackageIcons(softwareInstallers, resp.Packages), resp.DeletedPackages, resp.Categories, nil
|
|
default:
|
|
return nil, nil, nil, fmt.Errorf("unknown status: %q", resp.Status)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *Client) ListSelfServiceCategories(teamID uint) ([]fleet.SoftwareCategory, error) {
|
|
verb, path := "GET", "/api/latest/fleet/software/self_service_categories"
|
|
query := fmt.Sprintf("fleet_id=%d", teamID)
|
|
var responseBody getSelfServiceCategoriesResponse
|
|
if err := c.authenticatedRequestWithQuery(nil, verb, path, &responseBody, query); err != nil {
|
|
return nil, err
|
|
}
|
|
return responseBody.SelfServiceCategories, nil
|
|
}
|
|
|
|
func (c *Client) DeleteSelfServiceCategory(id uint) error {
|
|
verb, path := "DELETE", fmt.Sprintf("/api/latest/fleet/software/self_service_categories/%d", id)
|
|
var responseBody deleteSelfServiceCategoriesResponse
|
|
return c.authenticatedRequest(nil, verb, path, &responseBody)
|
|
}
|
|
|
|
// deleteUnusedSelfServiceCategories deletes the team's existing self-service categories that aren't
|
|
// in keep. Categories are created server-side by the software/VPP batch endpoints; keep is the
|
|
// union of categories those batches reported, so anything not in it is no longer needed
|
|
func (c *Client) deleteUnusedSelfServiceCategories(teamID uint, keep []string) error {
|
|
existing, err := c.ListSelfServiceCategories(teamID)
|
|
if err != nil {
|
|
return fmt.Errorf("listing existing self-service categories: %w", err)
|
|
}
|
|
for _, cat := range existing {
|
|
if slices.ContainsFunc(keep, func(name string) bool { return strings.EqualFold(name, cat.Name) }) {
|
|
continue
|
|
}
|
|
if err := c.DeleteSelfServiceCategory(cat.ID); err != nil {
|
|
return fmt.Errorf("deleting self-service category %q: %w", cat.Name, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// matchPackageIcons hydrates software responses with references to icons in the request payload, so we can track
|
|
// which API calls to make to add/update/delete icons
|
|
func matchPackageIcons(request []fleet.SoftwareInstallerPayload, response []fleet.SoftwarePackageResponse) []fleet.SoftwarePackageResponse {
|
|
// On the client side, software installer entries can have a URL or a hash or both ...
|
|
byURL := make(map[string]*fleet.SoftwareInstallerPayload)
|
|
byHash := make(map[string]*fleet.SoftwareInstallerPayload)
|
|
bySlug := make(map[string]*fleet.SoftwareInstallerPayload)
|
|
|
|
for i := range request {
|
|
clientSide := &request[i]
|
|
|
|
if clientSide.URL != "" {
|
|
byURL[clientSide.URL] = clientSide
|
|
}
|
|
if clientSide.SHA256 != "" {
|
|
byHash[clientSide.SHA256] = clientSide
|
|
}
|
|
if clientSide.Slug != nil {
|
|
bySlug[*clientSide.Slug] = clientSide
|
|
}
|
|
}
|
|
|
|
for i := range response {
|
|
serverSide := &response[i]
|
|
|
|
// All server side entries have a hash, so first try to match by that
|
|
if clientSide, ok := byHash[serverSide.HashSHA256]; ok {
|
|
serverSide.LocalIconHash = clientSide.IconHash
|
|
serverSide.LocalIconPath = clientSide.IconPath
|
|
continue
|
|
}
|
|
|
|
// ... Then by URL
|
|
if clientSide, ok := byURL[serverSide.URL]; ok {
|
|
serverSide.LocalIconHash = clientSide.IconHash
|
|
serverSide.LocalIconPath = clientSide.IconPath
|
|
continue
|
|
}
|
|
|
|
if clientSide, ok := bySlug[serverSide.Slug]; ok {
|
|
serverSide.LocalIconHash = clientSide.IconHash
|
|
serverSide.LocalIconPath = clientSide.IconPath
|
|
}
|
|
}
|
|
|
|
return response
|
|
}
|
|
|
|
func (c *Client) UploadIcon(teamID uint, titleID uint, filename string, iconReader io.Reader) error {
|
|
var buf bytes.Buffer
|
|
writer := multipart.NewWriter(&buf)
|
|
fileWriter, err := writer.CreateFormFile("icon", filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err = io.Copy(fileWriter, iconReader); err != nil {
|
|
return err
|
|
}
|
|
// Close the writer before using the buffer
|
|
if err := writer.Close(); err != nil {
|
|
return err
|
|
}
|
|
|
|
return c.putIcon(teamID, titleID, writer, buf)
|
|
}
|
|
|
|
func (c *Client) UpdateIcon(teamID uint, titleID uint, filename string, hash string) error {
|
|
var buf bytes.Buffer
|
|
writer := multipart.NewWriter(&buf)
|
|
if err := writer.WriteField("hash_sha256", hash); err != nil {
|
|
return err
|
|
}
|
|
if err := writer.WriteField("filename", filename); err != nil {
|
|
return err
|
|
}
|
|
// Close the writer before using the buffer
|
|
if err := writer.Close(); err != nil {
|
|
return err
|
|
}
|
|
|
|
return c.putIcon(teamID, titleID, writer, buf)
|
|
}
|
|
|
|
// ErrIconBytesMissing is returned by UpdateIcon when the server has the
|
|
// software_title_icons row but the underlying bytes for the requested storage
|
|
// hash are missing or fail integrity. Callers can fall back to a full upload
|
|
// to recover.
|
|
var ErrIconBytesMissing = errors.New("icon bytes missing on server")
|
|
|
|
func (c *Client) putIcon(teamID uint, titleID uint, writer *multipart.Writer, buf bytes.Buffer) error {
|
|
response, err := c.doContextWithBodyAndHeaders(
|
|
context.Background(),
|
|
"PUT",
|
|
fmt.Sprintf("/api/latest/fleet/software/titles/%d/icon", titleID),
|
|
fmt.Sprintf("fleet_id=%d", teamID),
|
|
buf.Bytes(),
|
|
map[string]string{
|
|
"Content-Type": writer.FormDataContentType(),
|
|
"Accept": "application/json",
|
|
"Authorization": fmt.Sprintf("Bearer %s", c.token),
|
|
},
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("do multipart request: %w", err)
|
|
}
|
|
defer response.Body.Close()
|
|
|
|
switch response.StatusCode {
|
|
case http.StatusOK:
|
|
return nil
|
|
case http.StatusConflict:
|
|
return ErrIconBytesMissing
|
|
default:
|
|
return fmt.Errorf("update icon: unexpected status code: %d", response.StatusCode)
|
|
}
|
|
}
|
|
|
|
func (c *Client) DeleteIcon(teamID uint, titleID uint) error {
|
|
response, err := c.AuthenticatedDo(
|
|
"DELETE",
|
|
fmt.Sprintf("/api/latest/fleet/software/titles/%d/icon", titleID),
|
|
fmt.Sprintf("fleet_id=%d", teamID),
|
|
nil,
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("delete icon: %w", err)
|
|
}
|
|
defer response.Body.Close()
|
|
|
|
if response.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("delete icon: unexpected status code: %d", response.StatusCode)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// InstallSoftware triggers a software installation (VPP or software package)
|
|
// on the specified host.
|
|
func (c *Client) InstallSoftware(hostID uint, softwareTitleID uint) error {
|
|
verb, path := "POST", fmt.Sprintf("/api/latest/fleet/hosts/%d/software/%d/install", hostID, softwareTitleID)
|
|
var responseBody installSoftwareResponse
|
|
return c.authenticatedRequest(nil, verb, path, &responseBody)
|
|
}
|
|
|
|
func (c *Client) GetFleetMaintainedApp(id uint) (*fleet.MaintainedApp, error) {
|
|
verb, path := "GET", fmt.Sprintf("/api/latest/fleet/software/fleet_maintained_apps/%d", id)
|
|
var responseBody getFleetMaintainedAppResponse
|
|
err := c.authenticatedRequest(nil, verb, path, &responseBody)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return responseBody.FleetMaintainedApp, nil
|
|
}
|
|
|
|
func (c *Client) ListFleetMaintainedApps(teamID uint) ([]fleet.MaintainedApp, error) {
|
|
verb, path := "GET", "/api/latest/fleet/software/fleet_maintained_apps"
|
|
query := fmt.Sprintf("fleet_id=%d", teamID)
|
|
|
|
var responseBody listFleetMaintainedAppsResponse
|
|
err := c.authenticatedRequestWithQuery(nil, verb, path, &responseBody, query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return responseBody.FleetMaintainedApps, nil
|
|
}
|