From 5c0dfbf6d936d51d0b06e01e6a7611ddfc4f338f Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Wed, 29 Jan 2025 10:24:44 -0600 Subject: [PATCH] Fleetd can now download software installers from signed CDN URLs. (#25276) For #24870 subtask API changes doc: #25293 # Checklist for submitter - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/Committing-Changes.md#changes-files) for more information. - [x] Added support on fleet's osquery simulator `cmd/osquery-perf` for new osquery data ingestion features. - [x] Added/updated automated tests - [x] Manual QA for all new/changed functionality - For Orbit and Fleet Desktop changes: - [x] Orbit runs on macOS, Linux and Windows. Check if the orbit feature/bugfix should only apply to one platform (`runtime.GOOS`). - [x] Manual QA must be performed in the three main OSs, macOS, Windows and Linux. - [x] Auto-update manual QA, from released version of component to new version (see [tools/tuf/test](../tools/tuf/test/README.md)). --- cmd/osquery-perf/agent.go | 253 +----------------- .../installer_cache/installer-cache.go | 45 +++- cmd/osquery-perf/osquery_perf/stats.go | 241 +++++++++++++++++ .../23823-download-software-from-cdn-url | 1 + orbit/pkg/installer/installer.go | 23 +- orbit/pkg/installer/installer_test.go | 211 ++++++++++++++- server/service/base_client.go | 21 +- server/service/orbit.go | 8 +- server/service/orbit_client.go | 59 ++-- 9 files changed, 576 insertions(+), 286 deletions(-) create mode 100644 cmd/osquery-perf/osquery_perf/stats.go create mode 100644 orbit/changes/23823-download-software-from-cdn-url diff --git a/cmd/osquery-perf/agent.go b/cmd/osquery-perf/agent.go index ce6b27fe51..3608403a09 100644 --- a/cmd/osquery-perf/agent.go +++ b/cmd/osquery-perf/agent.go @@ -28,6 +28,7 @@ import ( "time" "github.com/fleetdm/fleet/v4/cmd/osquery-perf/installer_cache" + "github.com/fleetdm/fleet/v4/cmd/osquery-perf/osquery_perf" "github.com/fleetdm/fleet/v4/pkg/file" "github.com/fleetdm/fleet/v4/pkg/mdm/mdmtest" "github.com/fleetdm/fleet/v4/server/fleet" @@ -143,240 +144,6 @@ func init() { ubuntuSoftware = loadSoftwareItems(ubuntuSoftwareFS, "ubuntu_2204-software.json.bz2", "deb_packages") } -type Stats struct { - startTime time.Time - errors int - osqueryEnrollments int - orbitEnrollments int - mdmEnrollments int - mdmSessions int - distributedWrites int - mdmCommandsReceived int - distributedReads int - configRequests int - configErrors int - resultLogRequests int - orbitErrors int - mdmErrors int - ddmDeclarationItemsErrors int - ddmConfigurationErrors int - ddmActivationErrors int - ddmStatusErrors int - ddmDeclarationItemsSuccess int - ddmConfigurationSuccess int - ddmActivationSuccess int - ddmStatusSuccess int - desktopErrors int - distributedReadErrors int - distributedWriteErrors int - resultLogErrors int - bufferedLogs int - - l sync.Mutex -} - -func (s *Stats) IncrementErrors(errors int) { - s.l.Lock() - defer s.l.Unlock() - s.errors += errors -} - -func (s *Stats) IncrementEnrollments() { - s.l.Lock() - defer s.l.Unlock() - s.osqueryEnrollments++ -} - -func (s *Stats) IncrementOrbitEnrollments() { - s.l.Lock() - defer s.l.Unlock() - s.orbitEnrollments++ -} - -func (s *Stats) IncrementMDMEnrollments() { - s.l.Lock() - defer s.l.Unlock() - s.mdmEnrollments++ -} - -func (s *Stats) IncrementMDMSessions() { - s.l.Lock() - defer s.l.Unlock() - s.mdmSessions++ -} - -func (s *Stats) IncrementDistributedWrites() { - s.l.Lock() - defer s.l.Unlock() - s.distributedWrites++ -} - -func (s *Stats) IncrementMDMCommandsReceived() { - s.l.Lock() - defer s.l.Unlock() - s.mdmCommandsReceived++ -} - -func (s *Stats) IncrementDistributedReads() { - s.l.Lock() - defer s.l.Unlock() - s.distributedReads++ -} - -func (s *Stats) IncrementConfigRequests() { - s.l.Lock() - defer s.l.Unlock() - s.configRequests++ -} - -func (s *Stats) IncrementConfigErrors() { - s.l.Lock() - defer s.l.Unlock() - s.configErrors++ -} - -func (s *Stats) IncrementResultLogRequests() { - s.l.Lock() - defer s.l.Unlock() - s.resultLogRequests++ -} - -func (s *Stats) IncrementOrbitErrors() { - s.l.Lock() - defer s.l.Unlock() - s.orbitErrors++ -} - -func (s *Stats) IncrementMDMErrors() { - s.l.Lock() - defer s.l.Unlock() - s.mdmErrors++ -} - -func (s *Stats) IncrementDDMDeclarationItemsErrors() { - s.l.Lock() - defer s.l.Unlock() - s.ddmDeclarationItemsErrors++ -} - -func (s *Stats) IncrementDDMConfigurationErrors() { - s.l.Lock() - defer s.l.Unlock() - s.ddmConfigurationErrors++ -} - -func (s *Stats) IncrementDDMActivationErrors() { - s.l.Lock() - defer s.l.Unlock() - s.ddmActivationErrors++ -} - -func (s *Stats) IncrementDDMStatusErrors() { - s.l.Lock() - defer s.l.Unlock() - s.ddmStatusErrors++ -} - -func (s *Stats) IncrementDDMDeclarationItemsSuccess() { - s.l.Lock() - defer s.l.Unlock() - s.ddmDeclarationItemsSuccess++ -} - -func (s *Stats) IncrementDDMConfigurationSuccess() { - s.l.Lock() - defer s.l.Unlock() - s.ddmConfigurationSuccess++ -} - -func (s *Stats) IncrementDDMActivationSuccess() { - s.l.Lock() - defer s.l.Unlock() - s.ddmActivationSuccess++ -} - -func (s *Stats) IncrementDDMStatusSuccess() { - s.l.Lock() - defer s.l.Unlock() - s.ddmStatusSuccess++ -} - -func (s *Stats) IncrementDesktopErrors() { - s.l.Lock() - defer s.l.Unlock() - s.desktopErrors++ -} - -func (s *Stats) IncrementDistributedReadErrors() { - s.l.Lock() - defer s.l.Unlock() - s.distributedReadErrors++ -} - -func (s *Stats) IncrementDistributedWriteErrors() { - s.l.Lock() - defer s.l.Unlock() - s.distributedWriteErrors++ -} - -func (s *Stats) IncrementResultLogErrors() { - s.l.Lock() - defer s.l.Unlock() - s.resultLogErrors++ -} - -func (s *Stats) UpdateBufferedLogs(v int) { - s.l.Lock() - defer s.l.Unlock() - s.bufferedLogs += v - if s.bufferedLogs < 0 { - s.bufferedLogs = 0 - } -} - -func (s *Stats) Log() { - s.l.Lock() - defer s.l.Unlock() - - log.Printf( - "uptime: %s, error rate: %.2f, osquery enrolls: %d, orbit enrolls: %d, mdm enrolls: %d, distributed/reads: %d, distributed/writes: %d, config requests: %d, result log requests: %d, mdm sessions initiated: %d, mdm commands received: %d, config errors: %d, distributed/read errors: %d, distributed/write errors: %d, log result errors: %d, orbit errors: %d, desktop errors: %d, mdm errors: %d, ddm declaration items success: %d, ddm declaration items errors: %d, ddm activation success: %d, ddm activation errors: %d, ddm configuration success: %d, ddm configuration errors: %d, ddm status success: %d, ddm status errors: %d, buffered logs: %d", - time.Since(s.startTime).Round(time.Second), - float64(s.errors)/float64(s.osqueryEnrollments), - s.osqueryEnrollments, - s.orbitEnrollments, - s.mdmEnrollments, - s.distributedReads, - s.distributedWrites, - s.configRequests, - s.resultLogRequests, - s.mdmSessions, - s.mdmCommandsReceived, - s.configErrors, - s.distributedReadErrors, - s.distributedWriteErrors, - s.resultLogErrors, - s.orbitErrors, - s.desktopErrors, - s.mdmErrors, - s.ddmDeclarationItemsSuccess, - s.ddmDeclarationItemsErrors, - s.ddmActivationSuccess, - s.ddmActivationErrors, - s.ddmConfigurationSuccess, - s.ddmConfigurationErrors, - s.ddmStatusSuccess, - s.ddmStatusErrors, - s.bufferedLogs, - ) -} - -func (s *Stats) runLoop() { - ticker := time.Tick(10 * time.Second) - for range ticker { - s.Log() - } -} - type nodeKeyManager struct { filepath string @@ -438,7 +205,7 @@ type mdmAgent struct { model string serverAddress string softwareCount softwareEntityCount - stats *Stats + stats *osquery_perf.Stats strings map[string]string } @@ -465,7 +232,7 @@ type agent struct { liveQueryNoResultsProb float64 strings map[string]string serverAddress string - stats *Stats + stats *osquery_perf.Stats nodeKeyManager *nodeKeyManager nodeKey string templates *template.Template @@ -1324,13 +1091,14 @@ func (a *agent) installSoftwareItem(installerID string, orbitClient *service.Orb if !failed { var cacheMiss bool // Download the file if needed to get its metadata - meta, cacheMiss, err = installerMetadataCache.Get(installer.InstallerID, orbitClient) + meta, cacheMiss, err = installerMetadataCache.Get(installer, orbitClient) if err != nil { return } - if !cacheMiss { - // If we didn't download and analyze the file, we do a download and don't save the result + if !cacheMiss && installer.SoftwareInstallerURL == nil { + // If we didn't download and analyze the file, AND we did not use a CDN URL to get the file, + // we do a download now and don't save the result. Doing this download adds realistic load on the server. err = orbitClient.DownloadAndDiscardSoftwareInstaller(installer.InstallerID) if err != nil { log.Println("download and discard software installer:", err) @@ -2803,10 +2571,11 @@ func main() { // Spread starts over the interval to prevent thundering herd sleepTime := *startPeriod / time.Duration(*hostCount) - stats := &Stats{ - startTime: time.Now(), + stats := &osquery_perf.Stats{ + StartTime: time.Now(), } - go stats.runLoop() + go stats.RunLoop() + installerMetadataCache.Stats = stats nodeKeyManager := &nodeKeyManager{} if nodeKeyFile != nil { diff --git a/cmd/osquery-perf/installer_cache/installer-cache.go b/cmd/osquery-perf/installer_cache/installer-cache.go index 2581528fab..f85d4a72c8 100644 --- a/cmd/osquery-perf/installer_cache/installer-cache.go +++ b/cmd/osquery-perf/installer_cache/installer-cache.go @@ -5,6 +5,7 @@ import ( "os" "sync" + "github.com/fleetdm/fleet/v4/cmd/osquery-perf/osquery_perf" "github.com/fleetdm/fleet/v4/pkg/file" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/service" @@ -16,9 +17,10 @@ import ( type Metadata struct { mu sync.Mutex cache map[uint]*file.InstallerMetadata + Stats *osquery_perf.Stats } -func (c *Metadata) Get(key uint, orbitClient *service.OrbitClient) (meta *file.InstallerMetadata, +func (c *Metadata) Get(installer *fleet.SoftwareInstallDetails, orbitClient *service.OrbitClient) (meta *file.InstallerMetadata, cacheMiss bool, err error) { c.mu.Lock() defer c.mu.Unlock() @@ -26,41 +28,60 @@ func (c *Metadata) Get(key uint, orbitClient *service.OrbitClient) (meta *file.I c.cache = make(map[uint]*file.InstallerMetadata, 1) } - meta, ok := c.cache[key] + meta, ok := c.cache[installer.InstallerID] if !ok { var err error - meta, err = populateMetadata(orbitClient, key) + meta, err = c.populateMetadata(installer, orbitClient) if err != nil { return nil, false, err } - c.cache[key] = meta + c.cache[installer.InstallerID] = meta cacheMiss = true } return meta, cacheMiss, nil } -func populateMetadata(orbitClient *service.OrbitClient, installerID uint) (*file.InstallerMetadata, error) { +func (c *Metadata) populateMetadata(installer *fleet.SoftwareInstallDetails, orbitClient *service.OrbitClient) (*file.InstallerMetadata, + error) { tmpDir, err := os.MkdirTemp("", "") if err != nil { - log.Println("create temp dir:", err) + c.Stats.IncrementOrbitErrors() + log.Println("level=error, create temp dir:", err) return nil, err } defer os.RemoveAll(tmpDir) - path, err := orbitClient.DownloadSoftwareInstaller(installerID, tmpDir) - if err != nil { - log.Println("download software installer:", err) - return nil, err + + var path string + if installer.SoftwareInstallerURL != nil { + path, err = orbitClient.DownloadSoftwareInstallerFromURL(installer.SoftwareInstallerURL.URL, + installer.SoftwareInstallerURL.Filename, tmpDir) + if err != nil { + log.Printf("level=error, msg=download software installer from URL; is CloudFront CDN configured correctly?, err=%s", err) + c.Stats.IncrementOrbitErrors() + return nil, err + } + } + + if path == "" { + path, err = orbitClient.DownloadSoftwareInstaller(installer.InstallerID, tmpDir) + if err != nil { + log.Printf("level=error, msg=download software installer, err=%s", err) + c.Stats.IncrementOrbitErrors() + return nil, err + } } // Figure out what we're actually installing here and add it to software inventory tfr, err := fleet.NewKeepFileReader(path) if err != nil { - log.Println("open installer:", err) + c.Stats.IncrementOrbitErrors() + log.Println("level=error, open installer:", err) return nil, err } defer tfr.Close() item, err := file.ExtractInstallerMetadata(tfr) if err != nil { - log.Println("extract installer metadata:", err) + c.Stats.IncrementOrbitErrors() + log.Println("level=error, extract installer metadata:", err) return nil, err } return item, nil diff --git a/cmd/osquery-perf/osquery_perf/stats.go b/cmd/osquery-perf/osquery_perf/stats.go new file mode 100644 index 0000000000..f20e851078 --- /dev/null +++ b/cmd/osquery-perf/osquery_perf/stats.go @@ -0,0 +1,241 @@ +package osquery_perf + +import ( + "log" + "sync" + "time" +) + +type Stats struct { + StartTime time.Time + errors int + osqueryEnrollments int + orbitEnrollments int + mdmEnrollments int + mdmSessions int + distributedWrites int + mdmCommandsReceived int + distributedReads int + configRequests int + configErrors int + resultLogRequests int + orbitErrors int + mdmErrors int + ddmDeclarationItemsErrors int + ddmConfigurationErrors int + ddmActivationErrors int + ddmStatusErrors int + ddmDeclarationItemsSuccess int + ddmConfigurationSuccess int + ddmActivationSuccess int + ddmStatusSuccess int + desktopErrors int + distributedReadErrors int + distributedWriteErrors int + resultLogErrors int + bufferedLogs int + + l sync.Mutex +} + +func (s *Stats) IncrementErrors(errors int) { + s.l.Lock() + defer s.l.Unlock() + s.errors += errors +} + +func (s *Stats) IncrementEnrollments() { + s.l.Lock() + defer s.l.Unlock() + s.osqueryEnrollments++ +} + +func (s *Stats) IncrementOrbitEnrollments() { + s.l.Lock() + defer s.l.Unlock() + s.orbitEnrollments++ +} + +func (s *Stats) IncrementMDMEnrollments() { + s.l.Lock() + defer s.l.Unlock() + s.mdmEnrollments++ +} + +func (s *Stats) IncrementMDMSessions() { + s.l.Lock() + defer s.l.Unlock() + s.mdmSessions++ +} + +func (s *Stats) IncrementDistributedWrites() { + s.l.Lock() + defer s.l.Unlock() + s.distributedWrites++ +} + +func (s *Stats) IncrementMDMCommandsReceived() { + s.l.Lock() + defer s.l.Unlock() + s.mdmCommandsReceived++ +} + +func (s *Stats) IncrementDistributedReads() { + s.l.Lock() + defer s.l.Unlock() + s.distributedReads++ +} + +func (s *Stats) IncrementConfigRequests() { + s.l.Lock() + defer s.l.Unlock() + s.configRequests++ +} + +func (s *Stats) IncrementConfigErrors() { + s.l.Lock() + defer s.l.Unlock() + s.configErrors++ +} + +func (s *Stats) IncrementResultLogRequests() { + s.l.Lock() + defer s.l.Unlock() + s.resultLogRequests++ +} + +func (s *Stats) IncrementOrbitErrors() { + s.l.Lock() + defer s.l.Unlock() + s.orbitErrors++ +} + +func (s *Stats) IncrementMDMErrors() { + s.l.Lock() + defer s.l.Unlock() + s.mdmErrors++ +} + +func (s *Stats) IncrementDDMDeclarationItemsErrors() { + s.l.Lock() + defer s.l.Unlock() + s.ddmDeclarationItemsErrors++ +} + +func (s *Stats) IncrementDDMConfigurationErrors() { + s.l.Lock() + defer s.l.Unlock() + s.ddmConfigurationErrors++ +} + +func (s *Stats) IncrementDDMActivationErrors() { + s.l.Lock() + defer s.l.Unlock() + s.ddmActivationErrors++ +} + +func (s *Stats) IncrementDDMStatusErrors() { + s.l.Lock() + defer s.l.Unlock() + s.ddmStatusErrors++ +} + +func (s *Stats) IncrementDDMDeclarationItemsSuccess() { + s.l.Lock() + defer s.l.Unlock() + s.ddmDeclarationItemsSuccess++ +} + +func (s *Stats) IncrementDDMConfigurationSuccess() { + s.l.Lock() + defer s.l.Unlock() + s.ddmConfigurationSuccess++ +} + +func (s *Stats) IncrementDDMActivationSuccess() { + s.l.Lock() + defer s.l.Unlock() + s.ddmActivationSuccess++ +} + +func (s *Stats) IncrementDDMStatusSuccess() { + s.l.Lock() + defer s.l.Unlock() + s.ddmStatusSuccess++ +} + +func (s *Stats) IncrementDesktopErrors() { + s.l.Lock() + defer s.l.Unlock() + s.desktopErrors++ +} + +func (s *Stats) IncrementDistributedReadErrors() { + s.l.Lock() + defer s.l.Unlock() + s.distributedReadErrors++ +} + +func (s *Stats) IncrementDistributedWriteErrors() { + s.l.Lock() + defer s.l.Unlock() + s.distributedWriteErrors++ +} + +func (s *Stats) IncrementResultLogErrors() { + s.l.Lock() + defer s.l.Unlock() + s.resultLogErrors++ +} + +func (s *Stats) UpdateBufferedLogs(v int) { + s.l.Lock() + defer s.l.Unlock() + s.bufferedLogs += v + if s.bufferedLogs < 0 { + s.bufferedLogs = 0 + } +} + +func (s *Stats) Log() { + s.l.Lock() + defer s.l.Unlock() + + log.Printf( + "uptime: %s, error rate: %.2f, osquery enrolls: %d, orbit enrolls: %d, mdm enrolls: %d, distributed/reads: %d, distributed/writes: %d, config requests: %d, result log requests: %d, mdm sessions initiated: %d, mdm commands received: %d, config errors: %d, distributed/read errors: %d, distributed/write errors: %d, log result errors: %d, orbit errors: %d, desktop errors: %d, mdm errors: %d, ddm declaration items success: %d, ddm declaration items errors: %d, ddm activation success: %d, ddm activation errors: %d, ddm configuration success: %d, ddm configuration errors: %d, ddm status success: %d, ddm status errors: %d, buffered logs: %d", + time.Since(s.StartTime).Round(time.Second), + float64(s.errors)/float64(s.osqueryEnrollments), + s.osqueryEnrollments, + s.orbitEnrollments, + s.mdmEnrollments, + s.distributedReads, + s.distributedWrites, + s.configRequests, + s.resultLogRequests, + s.mdmSessions, + s.mdmCommandsReceived, + s.configErrors, + s.distributedReadErrors, + s.distributedWriteErrors, + s.resultLogErrors, + s.orbitErrors, + s.desktopErrors, + s.mdmErrors, + s.ddmDeclarationItemsSuccess, + s.ddmDeclarationItemsErrors, + s.ddmActivationSuccess, + s.ddmActivationErrors, + s.ddmConfigurationSuccess, + s.ddmConfigurationErrors, + s.ddmStatusSuccess, + s.ddmStatusErrors, + s.bufferedLogs, + ) +} + +func (s *Stats) RunLoop() { + ticker := time.Tick(10 * time.Second) + for range ticker { + s.Log() + } +} diff --git a/orbit/changes/23823-download-software-from-cdn-url b/orbit/changes/23823-download-software-from-cdn-url new file mode 100644 index 0000000000..50d4680b30 --- /dev/null +++ b/orbit/changes/23823-download-software-from-cdn-url @@ -0,0 +1 @@ +Fleetd can now download software installers from signed CDN URLs. diff --git a/orbit/pkg/installer/installer.go b/orbit/pkg/installer/installer.go index b5544738eb..e4c672bf5f 100644 --- a/orbit/pkg/installer/installer.go +++ b/orbit/pkg/installer/installer.go @@ -34,6 +34,7 @@ type ( type Client interface { GetInstallerDetails(installID string) (*fleet.SoftwareInstallDetails, error) DownloadSoftwareInstaller(installerID uint, downloadDir string) (string, error) + DownloadSoftwareInstallerFromURL(url string, filename string, downloadDir string) (string, error) SaveInstallerResult(payload *fleet.HostSoftwareInstallResultPayload) error } @@ -220,10 +221,24 @@ func (r *Runner) installSoftware(ctx context.Context, installID string) (*fleet. return payload, fmt.Errorf("creating temporary directory: %w", err) } - log.Debug().Str("install_id", installID).Msgf("about to download software installer") - installerPath, err := r.OrbitClient.DownloadSoftwareInstaller(installer.InstallerID, tmpDir) - if err != nil { - return payload, err + var installerPath string + if installer.SoftwareInstallerURL != nil && installer.SoftwareInstallerURL.URL != "" { + log.Debug().Str("install_id", installID).Msgf("about to download software installer from URL") + installerPath, err = r.OrbitClient.DownloadSoftwareInstallerFromURL(installer.SoftwareInstallerURL.URL, + installer.SoftwareInstallerURL.Filename, tmpDir) + if err != nil { + log.Err(err).Msg("downloading software installer from URL") + // If download fails, we will fall back to downloading the installer directly from Fleet server + installerPath = "" + } + } + + if installerPath == "" { + log.Debug().Str("install_id", installID).Msgf("about to download software installer") + installerPath, err = r.OrbitClient.DownloadSoftwareInstaller(installer.InstallerID, tmpDir) + if err != nil { + return payload, err + } } // remove tmp directory and installer diff --git a/orbit/pkg/installer/installer_test.go b/orbit/pkg/installer/installer_test.go index c8dd78dc2e..7e1e5f60a5 100644 --- a/orbit/pkg/installer/installer_test.go +++ b/orbit/pkg/installer/installer_test.go @@ -19,9 +19,14 @@ import ( ) type TestOrbitClient struct { - downloadInstallerFn func(uint, string) (string, error) - getInstallerDetailsFn func(string) (*fleet.SoftwareInstallDetails, error) - saveInstallerResultFn func(*fleet.HostSoftwareInstallResultPayload) error + downloadInstallerFn func(uint, string) (string, error) + downloadInstallerFromURLFn func(url string, filename string, downloadDir string) (string, error) + getInstallerDetailsFn func(string) (*fleet.SoftwareInstallDetails, error) + saveInstallerResultFn func(*fleet.HostSoftwareInstallResultPayload) error +} + +func (oc *TestOrbitClient) DownloadSoftwareInstallerFromURL(url string, filename string, downloadDir string) (string, error) { + return oc.downloadInstallerFromURLFn(url, filename, downloadDir) } func (oc *TestOrbitClient) DownloadSoftwareInstaller(installerID uint, downloadDir string) (string, error) { @@ -415,6 +420,206 @@ func TestInstallerRun(t *testing.T) { }) } +func TestInstallerRunWithInstallerFromURL(t *testing.T) { + oc := &TestOrbitClient{} + + var getInstallerDetailsFnCalled bool + var installIdRequested string + installDetails := &fleet.SoftwareInstallDetails{ + ExecutionID: "exec1", + InstallerID: 1337, + InstallScript: "script1", + PostInstallScript: "script2", + SoftwareInstallerURL: &fleet.SoftwareInstallerURL{ + URL: "https://example.com/ABC", + Filename: "installer.pkg", + }, + } + getInstallerDetailsDefaultFn := func(installID string) (*fleet.SoftwareInstallDetails, error) { + getInstallerDetailsFnCalled = true + installIdRequested = installID + return installDetails, nil + } + oc.getInstallerDetailsFn = getInstallerDetailsDefaultFn + + var downloadInstallerFromURLFnCalled bool + downloadInstallerFromURLDefaultFn := func(url string, filename string, downloadDir string) (string, error) { + assert.Equal(t, installDetails.SoftwareInstallerURL.URL, url) + downloadInstallerFromURLFnCalled = true + return filepath.Join(downloadDir, filename), nil + } + oc.downloadInstallerFromURLFn = downloadInstallerFromURLDefaultFn + + var downloadInstallerFnCalled bool + downloadInstallerDefaultFn := func(installerID uint, downloadDir string) (string, error) { + downloadInstallerFnCalled = true + return filepath.Join(downloadDir, fmt.Sprint(installerID)+".pkg"), nil + } + oc.downloadInstallerFn = downloadInstallerDefaultFn + + var savedInstallerResult *fleet.HostSoftwareInstallResultPayload + oc.saveInstallerResultFn = func(hsirp *fleet.HostSoftwareInstallResultPayload) error { + savedInstallerResult = hsirp + return nil + } + + resetTestOrbitClient := func() { + getInstallerDetailsFnCalled = false + installIdRequested = "" + oc.getInstallerDetailsFn = getInstallerDetailsDefaultFn + installDetails = &fleet.SoftwareInstallDetails{ + ExecutionID: "exec1", + InstallerID: 1337, + InstallScript: "script1", + PostInstallScript: "script2", + SoftwareInstallerURL: &fleet.SoftwareInstallerURL{ + URL: "https://example.com/ABC", + Filename: "installer.pkg", + }, + } + downloadInstallerFnCalled = false + downloadInstallerFromURLFnCalled = false + oc.downloadInstallerFromURLFn = downloadInstallerFromURLDefaultFn + savedInstallerResult = nil + } + + r := &Runner{ + OrbitClient: oc, + scriptsEnabled: func() bool { return true }, + } + + var execCalled bool + var executedScripts []string + var execEnv []string + var execErr error + execOutput := []byte("execOutput") + execExitCode := 0 + execCmdDefaultFn := func(ctx context.Context, scriptPath string, env []string) ([]byte, int, error) { + execCalled = true + execEnv = env + executedScripts = append(executedScripts, scriptPath) + return execOutput, execExitCode, execErr + } + r.execCmdFn = execCmdDefaultFn + + var tmpDirFnCalled bool + var tmpDir string + r.tempDirFn = func(dir, pattern string) (string, error) { + tmpDirFnCalled = true + tmpDir = os.TempDir() + return tmpDir, nil + } + + var removeAllFnCalled bool + var removedDir string + r.removeAllFn = func(s string) error { + removedDir = s + removeAllFnCalled = true + return nil + } + + resetRunner := func() { + execCalled = false + executedScripts = nil + execEnv = nil + execOutput = []byte("execOutput") + execExitCode = 0 + execErr = nil + r.execCmdFn = execCmdDefaultFn + + tmpDirFnCalled = false + tmpDir = "" + } + + var config fleet.OrbitConfig + config.Notifications.PendingSoftwareInstallerIDs = []string{installDetails.ExecutionID} + + resetConfig := func() { + config.Notifications.PendingSoftwareInstallerIDs = []string{installDetails.ExecutionID} + } + + resetAll := func() { + resetTestOrbitClient() + resetRunner() + resetConfig() + } + + t.Run("everything good", func(t *testing.T) { + resetAll() + + err := r.run(context.Background(), &config) + require.NoError(t, err) + + assert.True(t, removeAllFnCalled) + assert.Equal(t, tmpDir, removedDir) + + assert.True(t, tmpDirFnCalled) + + assert.True(t, execCalled) + scriptExtension := ".sh" + if runtime.GOOS == "windows" { + scriptExtension = ".ps1" + } + assert.Contains(t, executedScripts, filepath.Join(tmpDir, "install-script"+scriptExtension)) + assert.Contains(t, executedScripts, filepath.Join(tmpDir, "post-install-script"+scriptExtension)) + assert.Contains(t, execEnv, "INSTALLER_PATH="+filepath.Join(tmpDir, installDetails.SoftwareInstallerURL.Filename)) + + assert.NotNil(t, savedInstallerResult) + assert.Equal(t, execExitCode, *savedInstallerResult.InstallScriptExitCode) + assert.Equal(t, string(execOutput), *savedInstallerResult.InstallScriptOutput) + assert.Equal(t, execExitCode, *savedInstallerResult.PostInstallScriptExitCode) + assert.Equal(t, string(execOutput), *savedInstallerResult.PostInstallScriptOutput) + assert.Equal(t, installDetails.ExecutionID, savedInstallerResult.InstallUUID) + + assert.True(t, downloadInstallerFromURLFnCalled) + assert.False(t, downloadInstallerFnCalled) + + assert.True(t, getInstallerDetailsFnCalled) + assert.Equal(t, installDetails.ExecutionID, installIdRequested) + }) + + t.Run("CDN fails and we fall back to Fleet download", func(t *testing.T) { + resetAll() + + oc.downloadInstallerFromURLFn = func(url string, filename string, downloadDir string) (string, error) { + assert.Equal(t, installDetails.SoftwareInstallerURL.URL, url) + downloadInstallerFromURLFnCalled = true + return "bozo", errors.New("test error") + } + + err := r.run(context.Background(), &config) + require.NoError(t, err) + + assert.True(t, removeAllFnCalled) + assert.Equal(t, tmpDir, removedDir) + + assert.True(t, tmpDirFnCalled) + + assert.True(t, execCalled) + scriptExtension := ".sh" + if runtime.GOOS == "windows" { + scriptExtension = ".ps1" + } + assert.Contains(t, executedScripts, filepath.Join(tmpDir, "install-script"+scriptExtension)) + assert.Contains(t, executedScripts, filepath.Join(tmpDir, "post-install-script"+scriptExtension)) + require.Contains(t, execEnv, "INSTALLER_PATH="+filepath.Join(tmpDir, fmt.Sprint(installDetails.InstallerID)+".pkg")) + + assert.NotNil(t, savedInstallerResult) + assert.Equal(t, execExitCode, *savedInstallerResult.InstallScriptExitCode) + assert.Equal(t, string(execOutput), *savedInstallerResult.InstallScriptOutput) + assert.Equal(t, execExitCode, *savedInstallerResult.PostInstallScriptExitCode) + assert.Equal(t, string(execOutput), *savedInstallerResult.PostInstallScriptOutput) + assert.Equal(t, installDetails.ExecutionID, savedInstallerResult.InstallUUID) + + assert.True(t, downloadInstallerFromURLFnCalled) + assert.True(t, downloadInstallerFnCalled) + + assert.True(t, getInstallerDetailsFnCalled) + assert.Equal(t, installDetails.ExecutionID, installIdRequested) + }) + +} + func TestScriptsDisabled(t *testing.T) { oc := &TestOrbitClient{} qc := &TestQueryClient{} diff --git a/server/service/base_client.go b/server/service/base_client.go index da1ad82034..bb33559e92 100644 --- a/server/service/base_client.go +++ b/server/service/base_client.go @@ -196,18 +196,25 @@ type bodyHandler interface { } type FileResponse struct { - DestPath string - DestFile string - destFilePath string + DestPath string + DestFile string + destFilePath string + SkipMediaType bool } func (f *FileResponse) Handle(resp *http.Response) error { - _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")) - if err != nil { - return fmt.Errorf("parsing media type from response header: %w", err) + var filename string + if !f.SkipMediaType { + _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")) + if err != nil { + return fmt.Errorf("parsing media type from response header: %w", err) + } + filename = params["filename"] } - filename := params["filename"] + if filename == "" { + filename = f.DestFile + } if filename == "" { filename = uuid.NewString() } diff --git a/server/service/orbit.go b/server/service/orbit.go index 923320b279..21502285cb 100644 --- a/server/service/orbit.go +++ b/server/service/orbit.go @@ -1138,7 +1138,8 @@ func (svc *Service) validateAndEncrypt(ctx context.Context, passphrase string, s ///////////////////////////////////////////////////////////////////////////////// type orbitGetSoftwareInstallRequest struct { - OrbitNodeKey string `json:"orbot_node_key"` + OrbitNodeKey string `json:"orbit_node_key"` + OrbotNodeKey string `json:"orbot_node_key"` // legacy typo -- keep for backwards compatibility with orbit <= 1.38.0 InstallUUID string `json:"install_uuid"` } @@ -1149,7 +1150,10 @@ func (r *orbitGetSoftwareInstallRequest) setOrbitNodeKey(nodeKey string) { // interface implementation required by the OrbitClient func (r *orbitGetSoftwareInstallRequest) orbitHostNodeKey() string { - return r.OrbitNodeKey + if r.OrbitNodeKey != "" { + return r.OrbitNodeKey + } + return r.OrbotNodeKey } type orbitGetSoftwareInstallResponse struct { diff --git a/server/service/orbit_client.go b/server/service/orbit_client.go index 912d47c86f..ca18f81d17 100644 --- a/server/service/orbit_client.go +++ b/server/service/orbit_client.go @@ -73,6 +73,12 @@ type configCache struct { } func (oc *OrbitClient) request(verb string, path string, params interface{}, resp interface{}) error { + return oc.requestWithExternal(verb, path, params, resp, false) +} + +// requestWithExternal is used to make requests to Fleet or external URLs. If external is true, the pathOrURL +// is used as the full URL to make the request to. +func (oc *OrbitClient) requestWithExternal(verb string, pathOrURL string, params interface{}, resp interface{}, external bool) error { var bodyBytes []byte var err error if params != nil { @@ -82,11 +88,6 @@ func (oc *OrbitClient) request(verb string, path string, params interface{}, res } } - parsedURL, err := url.Parse(path) - if err != nil { - return fmt.Errorf("parsing URL: %w", err) - } - oc.closeIdleConnections() ctx := context.Background() @@ -94,24 +95,42 @@ func (oc *OrbitClient) request(verb string, path string, params interface{}, res ctx = httptrace.WithClientTrace(ctx, testStdoutHTTPTracer) } - request, err := http.NewRequestWithContext( - ctx, - verb, - oc.url(parsedURL.Path, parsedURL.RawQuery).String(), - bytes.NewBuffer(bodyBytes), - ) - if err != nil { - return err + var request *http.Request + if external { + request, err = http.NewRequestWithContext( + ctx, + verb, + pathOrURL, + nil, + ) + if err != nil { + return err + } + } else { + parsedURL, err := url.Parse(pathOrURL) + if err != nil { + return fmt.Errorf("parsing URL: %w", err) + } + + request, err = http.NewRequestWithContext( + ctx, + verb, + oc.url(parsedURL.Path, parsedURL.RawQuery).String(), + bytes.NewBuffer(bodyBytes), + ) + if err != nil { + return err + } + oc.setClientCapabilitiesHeader(request) } - oc.setClientCapabilitiesHeader(request) response, err := oc.http.Do(request) if err != nil { oc.setLastRecordedError(err) - return fmt.Errorf("%s %s: %w", verb, path, err) + return fmt.Errorf("%s %s: %w", verb, pathOrURL, err) } defer response.Body.Close() - if err := oc.parseResponse(verb, path, response, resp); err != nil { + if err := oc.parseResponse(verb, pathOrURL, response, resp); err != nil { oc.setLastRecordedError(err) return err } @@ -413,6 +432,14 @@ func (oc *OrbitClient) DownloadSoftwareInstaller(installerID uint, downloadDirec return resp.GetFilePath(), nil } +func (oc *OrbitClient) DownloadSoftwareInstallerFromURL(url string, filename string, downloadDirectory string) (string, error) { + resp := FileResponse{DestPath: downloadDirectory, DestFile: filename, SkipMediaType: true} + if err := oc.requestWithExternal("GET", url, nil, &resp, true); err != nil { + return "", err + } + return resp.GetFilePath(), nil +} + type NullFileResponse struct{} func (f *NullFileResponse) Handle(resp *http.Response) error {