Fleetd can now download software installers from signed CDN URLs. (#25276)
For #24870 subtask API changes doc: #25293 # Checklist for submitter - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/Committing-Changes.md#changes-files) for more information. - [x] Added support on fleet's osquery simulator `cmd/osquery-perf` for new osquery data ingestion features. - [x] Added/updated automated tests - [x] Manual QA for all new/changed functionality - For Orbit and Fleet Desktop changes: - [x] Orbit runs on macOS, Linux and Windows. Check if the orbit feature/bugfix should only apply to one platform (`runtime.GOOS`). - [x] Manual QA must be performed in the three main OSs, macOS, Windows and Linux. - [x] Auto-update manual QA, from released version of component to new version (see [tools/tuf/test](../tools/tuf/test/README.md)).
This commit is contained in:
Victor Lyuboslavsky
+11
-242
@@ -28,6 +28,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/fleetdm/fleet/v4/cmd/osquery-perf/installer_cache"
|
||||
"github.com/fleetdm/fleet/v4/cmd/osquery-perf/osquery_perf"
|
||||
"github.com/fleetdm/fleet/v4/pkg/file"
|
||||
"github.com/fleetdm/fleet/v4/pkg/mdm/mdmtest"
|
||||
"github.com/fleetdm/fleet/v4/server/fleet"
|
||||
@@ -143,240 +144,6 @@ func init() {
|
||||
ubuntuSoftware = loadSoftwareItems(ubuntuSoftwareFS, "ubuntu_2204-software.json.bz2", "deb_packages")
|
||||
}
|
||||
|
||||
type Stats struct {
|
||||
startTime time.Time
|
||||
errors int
|
||||
osqueryEnrollments int
|
||||
orbitEnrollments int
|
||||
mdmEnrollments int
|
||||
mdmSessions int
|
||||
distributedWrites int
|
||||
mdmCommandsReceived int
|
||||
distributedReads int
|
||||
configRequests int
|
||||
configErrors int
|
||||
resultLogRequests int
|
||||
orbitErrors int
|
||||
mdmErrors int
|
||||
ddmDeclarationItemsErrors int
|
||||
ddmConfigurationErrors int
|
||||
ddmActivationErrors int
|
||||
ddmStatusErrors int
|
||||
ddmDeclarationItemsSuccess int
|
||||
ddmConfigurationSuccess int
|
||||
ddmActivationSuccess int
|
||||
ddmStatusSuccess int
|
||||
desktopErrors int
|
||||
distributedReadErrors int
|
||||
distributedWriteErrors int
|
||||
resultLogErrors int
|
||||
bufferedLogs int
|
||||
|
||||
l sync.Mutex
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementErrors(errors int) {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.errors += errors
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementEnrollments() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.osqueryEnrollments++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementOrbitEnrollments() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.orbitEnrollments++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementMDMEnrollments() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.mdmEnrollments++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementMDMSessions() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.mdmSessions++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDistributedWrites() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.distributedWrites++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementMDMCommandsReceived() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.mdmCommandsReceived++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDistributedReads() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.distributedReads++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementConfigRequests() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.configRequests++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementConfigErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.configErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementResultLogRequests() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.resultLogRequests++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementOrbitErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.orbitErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementMDMErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.mdmErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMDeclarationItemsErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmDeclarationItemsErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMConfigurationErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmConfigurationErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMActivationErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmActivationErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMStatusErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmStatusErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMDeclarationItemsSuccess() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmDeclarationItemsSuccess++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMConfigurationSuccess() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmConfigurationSuccess++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMActivationSuccess() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmActivationSuccess++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMStatusSuccess() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmStatusSuccess++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDesktopErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.desktopErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDistributedReadErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.distributedReadErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDistributedWriteErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.distributedWriteErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementResultLogErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.resultLogErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) UpdateBufferedLogs(v int) {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.bufferedLogs += v
|
||||
if s.bufferedLogs < 0 {
|
||||
s.bufferedLogs = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Stats) Log() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
|
||||
log.Printf(
|
||||
"uptime: %s, error rate: %.2f, osquery enrolls: %d, orbit enrolls: %d, mdm enrolls: %d, distributed/reads: %d, distributed/writes: %d, config requests: %d, result log requests: %d, mdm sessions initiated: %d, mdm commands received: %d, config errors: %d, distributed/read errors: %d, distributed/write errors: %d, log result errors: %d, orbit errors: %d, desktop errors: %d, mdm errors: %d, ddm declaration items success: %d, ddm declaration items errors: %d, ddm activation success: %d, ddm activation errors: %d, ddm configuration success: %d, ddm configuration errors: %d, ddm status success: %d, ddm status errors: %d, buffered logs: %d",
|
||||
time.Since(s.startTime).Round(time.Second),
|
||||
float64(s.errors)/float64(s.osqueryEnrollments),
|
||||
s.osqueryEnrollments,
|
||||
s.orbitEnrollments,
|
||||
s.mdmEnrollments,
|
||||
s.distributedReads,
|
||||
s.distributedWrites,
|
||||
s.configRequests,
|
||||
s.resultLogRequests,
|
||||
s.mdmSessions,
|
||||
s.mdmCommandsReceived,
|
||||
s.configErrors,
|
||||
s.distributedReadErrors,
|
||||
s.distributedWriteErrors,
|
||||
s.resultLogErrors,
|
||||
s.orbitErrors,
|
||||
s.desktopErrors,
|
||||
s.mdmErrors,
|
||||
s.ddmDeclarationItemsSuccess,
|
||||
s.ddmDeclarationItemsErrors,
|
||||
s.ddmActivationSuccess,
|
||||
s.ddmActivationErrors,
|
||||
s.ddmConfigurationSuccess,
|
||||
s.ddmConfigurationErrors,
|
||||
s.ddmStatusSuccess,
|
||||
s.ddmStatusErrors,
|
||||
s.bufferedLogs,
|
||||
)
|
||||
}
|
||||
|
||||
func (s *Stats) runLoop() {
|
||||
ticker := time.Tick(10 * time.Second)
|
||||
for range ticker {
|
||||
s.Log()
|
||||
}
|
||||
}
|
||||
|
||||
type nodeKeyManager struct {
|
||||
filepath string
|
||||
|
||||
@@ -438,7 +205,7 @@ type mdmAgent struct {
|
||||
model string
|
||||
serverAddress string
|
||||
softwareCount softwareEntityCount
|
||||
stats *Stats
|
||||
stats *osquery_perf.Stats
|
||||
strings map[string]string
|
||||
}
|
||||
|
||||
@@ -465,7 +232,7 @@ type agent struct {
|
||||
liveQueryNoResultsProb float64
|
||||
strings map[string]string
|
||||
serverAddress string
|
||||
stats *Stats
|
||||
stats *osquery_perf.Stats
|
||||
nodeKeyManager *nodeKeyManager
|
||||
nodeKey string
|
||||
templates *template.Template
|
||||
@@ -1324,13 +1091,14 @@ func (a *agent) installSoftwareItem(installerID string, orbitClient *service.Orb
|
||||
if !failed {
|
||||
var cacheMiss bool
|
||||
// Download the file if needed to get its metadata
|
||||
meta, cacheMiss, err = installerMetadataCache.Get(installer.InstallerID, orbitClient)
|
||||
meta, cacheMiss, err = installerMetadataCache.Get(installer, orbitClient)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if !cacheMiss {
|
||||
// If we didn't download and analyze the file, we do a download and don't save the result
|
||||
if !cacheMiss && installer.SoftwareInstallerURL == nil {
|
||||
// If we didn't download and analyze the file, AND we did not use a CDN URL to get the file,
|
||||
// we do a download now and don't save the result. Doing this download adds realistic load on the server.
|
||||
err = orbitClient.DownloadAndDiscardSoftwareInstaller(installer.InstallerID)
|
||||
if err != nil {
|
||||
log.Println("download and discard software installer:", err)
|
||||
@@ -2803,10 +2571,11 @@ func main() {
|
||||
// Spread starts over the interval to prevent thundering herd
|
||||
sleepTime := *startPeriod / time.Duration(*hostCount)
|
||||
|
||||
stats := &Stats{
|
||||
startTime: time.Now(),
|
||||
stats := &osquery_perf.Stats{
|
||||
StartTime: time.Now(),
|
||||
}
|
||||
go stats.runLoop()
|
||||
go stats.RunLoop()
|
||||
installerMetadataCache.Stats = stats
|
||||
|
||||
nodeKeyManager := &nodeKeyManager{}
|
||||
if nodeKeyFile != nil {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/fleetdm/fleet/v4/cmd/osquery-perf/osquery_perf"
|
||||
"github.com/fleetdm/fleet/v4/pkg/file"
|
||||
"github.com/fleetdm/fleet/v4/server/fleet"
|
||||
"github.com/fleetdm/fleet/v4/server/service"
|
||||
@@ -16,9 +17,10 @@ import (
|
||||
type Metadata struct {
|
||||
mu sync.Mutex
|
||||
cache map[uint]*file.InstallerMetadata
|
||||
Stats *osquery_perf.Stats
|
||||
}
|
||||
|
||||
func (c *Metadata) Get(key uint, orbitClient *service.OrbitClient) (meta *file.InstallerMetadata,
|
||||
func (c *Metadata) Get(installer *fleet.SoftwareInstallDetails, orbitClient *service.OrbitClient) (meta *file.InstallerMetadata,
|
||||
cacheMiss bool, err error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
@@ -26,41 +28,60 @@ func (c *Metadata) Get(key uint, orbitClient *service.OrbitClient) (meta *file.I
|
||||
c.cache = make(map[uint]*file.InstallerMetadata, 1)
|
||||
}
|
||||
|
||||
meta, ok := c.cache[key]
|
||||
meta, ok := c.cache[installer.InstallerID]
|
||||
if !ok {
|
||||
var err error
|
||||
meta, err = populateMetadata(orbitClient, key)
|
||||
meta, err = c.populateMetadata(installer, orbitClient)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
c.cache[key] = meta
|
||||
c.cache[installer.InstallerID] = meta
|
||||
cacheMiss = true
|
||||
}
|
||||
return meta, cacheMiss, nil
|
||||
}
|
||||
|
||||
func populateMetadata(orbitClient *service.OrbitClient, installerID uint) (*file.InstallerMetadata, error) {
|
||||
func (c *Metadata) populateMetadata(installer *fleet.SoftwareInstallDetails, orbitClient *service.OrbitClient) (*file.InstallerMetadata,
|
||||
error) {
|
||||
tmpDir, err := os.MkdirTemp("", "")
|
||||
if err != nil {
|
||||
log.Println("create temp dir:", err)
|
||||
c.Stats.IncrementOrbitErrors()
|
||||
log.Println("level=error, create temp dir:", err)
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
path, err := orbitClient.DownloadSoftwareInstaller(installerID, tmpDir)
|
||||
if err != nil {
|
||||
log.Println("download software installer:", err)
|
||||
return nil, err
|
||||
|
||||
var path string
|
||||
if installer.SoftwareInstallerURL != nil {
|
||||
path, err = orbitClient.DownloadSoftwareInstallerFromURL(installer.SoftwareInstallerURL.URL,
|
||||
installer.SoftwareInstallerURL.Filename, tmpDir)
|
||||
if err != nil {
|
||||
log.Printf("level=error, msg=download software installer from URL; is CloudFront CDN configured correctly?, err=%s", err)
|
||||
c.Stats.IncrementOrbitErrors()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if path == "" {
|
||||
path, err = orbitClient.DownloadSoftwareInstaller(installer.InstallerID, tmpDir)
|
||||
if err != nil {
|
||||
log.Printf("level=error, msg=download software installer, err=%s", err)
|
||||
c.Stats.IncrementOrbitErrors()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// Figure out what we're actually installing here and add it to software inventory
|
||||
tfr, err := fleet.NewKeepFileReader(path)
|
||||
if err != nil {
|
||||
log.Println("open installer:", err)
|
||||
c.Stats.IncrementOrbitErrors()
|
||||
log.Println("level=error, open installer:", err)
|
||||
return nil, err
|
||||
}
|
||||
defer tfr.Close()
|
||||
item, err := file.ExtractInstallerMetadata(tfr)
|
||||
if err != nil {
|
||||
log.Println("extract installer metadata:", err)
|
||||
c.Stats.IncrementOrbitErrors()
|
||||
log.Println("level=error, extract installer metadata:", err)
|
||||
return nil, err
|
||||
}
|
||||
return item, nil
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
package osquery_perf
|
||||
|
||||
import (
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Stats struct {
|
||||
StartTime time.Time
|
||||
errors int
|
||||
osqueryEnrollments int
|
||||
orbitEnrollments int
|
||||
mdmEnrollments int
|
||||
mdmSessions int
|
||||
distributedWrites int
|
||||
mdmCommandsReceived int
|
||||
distributedReads int
|
||||
configRequests int
|
||||
configErrors int
|
||||
resultLogRequests int
|
||||
orbitErrors int
|
||||
mdmErrors int
|
||||
ddmDeclarationItemsErrors int
|
||||
ddmConfigurationErrors int
|
||||
ddmActivationErrors int
|
||||
ddmStatusErrors int
|
||||
ddmDeclarationItemsSuccess int
|
||||
ddmConfigurationSuccess int
|
||||
ddmActivationSuccess int
|
||||
ddmStatusSuccess int
|
||||
desktopErrors int
|
||||
distributedReadErrors int
|
||||
distributedWriteErrors int
|
||||
resultLogErrors int
|
||||
bufferedLogs int
|
||||
|
||||
l sync.Mutex
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementErrors(errors int) {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.errors += errors
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementEnrollments() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.osqueryEnrollments++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementOrbitEnrollments() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.orbitEnrollments++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementMDMEnrollments() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.mdmEnrollments++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementMDMSessions() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.mdmSessions++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDistributedWrites() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.distributedWrites++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementMDMCommandsReceived() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.mdmCommandsReceived++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDistributedReads() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.distributedReads++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementConfigRequests() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.configRequests++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementConfigErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.configErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementResultLogRequests() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.resultLogRequests++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementOrbitErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.orbitErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementMDMErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.mdmErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMDeclarationItemsErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmDeclarationItemsErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMConfigurationErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmConfigurationErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMActivationErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmActivationErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMStatusErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmStatusErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMDeclarationItemsSuccess() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmDeclarationItemsSuccess++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMConfigurationSuccess() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmConfigurationSuccess++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMActivationSuccess() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmActivationSuccess++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDDMStatusSuccess() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.ddmStatusSuccess++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDesktopErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.desktopErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDistributedReadErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.distributedReadErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementDistributedWriteErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.distributedWriteErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) IncrementResultLogErrors() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.resultLogErrors++
|
||||
}
|
||||
|
||||
func (s *Stats) UpdateBufferedLogs(v int) {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
s.bufferedLogs += v
|
||||
if s.bufferedLogs < 0 {
|
||||
s.bufferedLogs = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Stats) Log() {
|
||||
s.l.Lock()
|
||||
defer s.l.Unlock()
|
||||
|
||||
log.Printf(
|
||||
"uptime: %s, error rate: %.2f, osquery enrolls: %d, orbit enrolls: %d, mdm enrolls: %d, distributed/reads: %d, distributed/writes: %d, config requests: %d, result log requests: %d, mdm sessions initiated: %d, mdm commands received: %d, config errors: %d, distributed/read errors: %d, distributed/write errors: %d, log result errors: %d, orbit errors: %d, desktop errors: %d, mdm errors: %d, ddm declaration items success: %d, ddm declaration items errors: %d, ddm activation success: %d, ddm activation errors: %d, ddm configuration success: %d, ddm configuration errors: %d, ddm status success: %d, ddm status errors: %d, buffered logs: %d",
|
||||
time.Since(s.StartTime).Round(time.Second),
|
||||
float64(s.errors)/float64(s.osqueryEnrollments),
|
||||
s.osqueryEnrollments,
|
||||
s.orbitEnrollments,
|
||||
s.mdmEnrollments,
|
||||
s.distributedReads,
|
||||
s.distributedWrites,
|
||||
s.configRequests,
|
||||
s.resultLogRequests,
|
||||
s.mdmSessions,
|
||||
s.mdmCommandsReceived,
|
||||
s.configErrors,
|
||||
s.distributedReadErrors,
|
||||
s.distributedWriteErrors,
|
||||
s.resultLogErrors,
|
||||
s.orbitErrors,
|
||||
s.desktopErrors,
|
||||
s.mdmErrors,
|
||||
s.ddmDeclarationItemsSuccess,
|
||||
s.ddmDeclarationItemsErrors,
|
||||
s.ddmActivationSuccess,
|
||||
s.ddmActivationErrors,
|
||||
s.ddmConfigurationSuccess,
|
||||
s.ddmConfigurationErrors,
|
||||
s.ddmStatusSuccess,
|
||||
s.ddmStatusErrors,
|
||||
s.bufferedLogs,
|
||||
)
|
||||
}
|
||||
|
||||
func (s *Stats) RunLoop() {
|
||||
ticker := time.Tick(10 * time.Second)
|
||||
for range ticker {
|
||||
s.Log()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
Fleetd can now download software installers from signed CDN URLs.
|
||||
@@ -34,6 +34,7 @@ type (
|
||||
type Client interface {
|
||||
GetInstallerDetails(installID string) (*fleet.SoftwareInstallDetails, error)
|
||||
DownloadSoftwareInstaller(installerID uint, downloadDir string) (string, error)
|
||||
DownloadSoftwareInstallerFromURL(url string, filename string, downloadDir string) (string, error)
|
||||
SaveInstallerResult(payload *fleet.HostSoftwareInstallResultPayload) error
|
||||
}
|
||||
|
||||
@@ -220,10 +221,24 @@ func (r *Runner) installSoftware(ctx context.Context, installID string) (*fleet.
|
||||
return payload, fmt.Errorf("creating temporary directory: %w", err)
|
||||
}
|
||||
|
||||
log.Debug().Str("install_id", installID).Msgf("about to download software installer")
|
||||
installerPath, err := r.OrbitClient.DownloadSoftwareInstaller(installer.InstallerID, tmpDir)
|
||||
if err != nil {
|
||||
return payload, err
|
||||
var installerPath string
|
||||
if installer.SoftwareInstallerURL != nil && installer.SoftwareInstallerURL.URL != "" {
|
||||
log.Debug().Str("install_id", installID).Msgf("about to download software installer from URL")
|
||||
installerPath, err = r.OrbitClient.DownloadSoftwareInstallerFromURL(installer.SoftwareInstallerURL.URL,
|
||||
installer.SoftwareInstallerURL.Filename, tmpDir)
|
||||
if err != nil {
|
||||
log.Err(err).Msg("downloading software installer from URL")
|
||||
// If download fails, we will fall back to downloading the installer directly from Fleet server
|
||||
installerPath = ""
|
||||
}
|
||||
}
|
||||
|
||||
if installerPath == "" {
|
||||
log.Debug().Str("install_id", installID).Msgf("about to download software installer")
|
||||
installerPath, err = r.OrbitClient.DownloadSoftwareInstaller(installer.InstallerID, tmpDir)
|
||||
if err != nil {
|
||||
return payload, err
|
||||
}
|
||||
}
|
||||
|
||||
// remove tmp directory and installer
|
||||
|
||||
@@ -19,9 +19,14 @@ import (
|
||||
)
|
||||
|
||||
type TestOrbitClient struct {
|
||||
downloadInstallerFn func(uint, string) (string, error)
|
||||
getInstallerDetailsFn func(string) (*fleet.SoftwareInstallDetails, error)
|
||||
saveInstallerResultFn func(*fleet.HostSoftwareInstallResultPayload) error
|
||||
downloadInstallerFn func(uint, string) (string, error)
|
||||
downloadInstallerFromURLFn func(url string, filename string, downloadDir string) (string, error)
|
||||
getInstallerDetailsFn func(string) (*fleet.SoftwareInstallDetails, error)
|
||||
saveInstallerResultFn func(*fleet.HostSoftwareInstallResultPayload) error
|
||||
}
|
||||
|
||||
func (oc *TestOrbitClient) DownloadSoftwareInstallerFromURL(url string, filename string, downloadDir string) (string, error) {
|
||||
return oc.downloadInstallerFromURLFn(url, filename, downloadDir)
|
||||
}
|
||||
|
||||
func (oc *TestOrbitClient) DownloadSoftwareInstaller(installerID uint, downloadDir string) (string, error) {
|
||||
@@ -415,6 +420,206 @@ func TestInstallerRun(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestInstallerRunWithInstallerFromURL(t *testing.T) {
|
||||
oc := &TestOrbitClient{}
|
||||
|
||||
var getInstallerDetailsFnCalled bool
|
||||
var installIdRequested string
|
||||
installDetails := &fleet.SoftwareInstallDetails{
|
||||
ExecutionID: "exec1",
|
||||
InstallerID: 1337,
|
||||
InstallScript: "script1",
|
||||
PostInstallScript: "script2",
|
||||
SoftwareInstallerURL: &fleet.SoftwareInstallerURL{
|
||||
URL: "https://example.com/ABC",
|
||||
Filename: "installer.pkg",
|
||||
},
|
||||
}
|
||||
getInstallerDetailsDefaultFn := func(installID string) (*fleet.SoftwareInstallDetails, error) {
|
||||
getInstallerDetailsFnCalled = true
|
||||
installIdRequested = installID
|
||||
return installDetails, nil
|
||||
}
|
||||
oc.getInstallerDetailsFn = getInstallerDetailsDefaultFn
|
||||
|
||||
var downloadInstallerFromURLFnCalled bool
|
||||
downloadInstallerFromURLDefaultFn := func(url string, filename string, downloadDir string) (string, error) {
|
||||
assert.Equal(t, installDetails.SoftwareInstallerURL.URL, url)
|
||||
downloadInstallerFromURLFnCalled = true
|
||||
return filepath.Join(downloadDir, filename), nil
|
||||
}
|
||||
oc.downloadInstallerFromURLFn = downloadInstallerFromURLDefaultFn
|
||||
|
||||
var downloadInstallerFnCalled bool
|
||||
downloadInstallerDefaultFn := func(installerID uint, downloadDir string) (string, error) {
|
||||
downloadInstallerFnCalled = true
|
||||
return filepath.Join(downloadDir, fmt.Sprint(installerID)+".pkg"), nil
|
||||
}
|
||||
oc.downloadInstallerFn = downloadInstallerDefaultFn
|
||||
|
||||
var savedInstallerResult *fleet.HostSoftwareInstallResultPayload
|
||||
oc.saveInstallerResultFn = func(hsirp *fleet.HostSoftwareInstallResultPayload) error {
|
||||
savedInstallerResult = hsirp
|
||||
return nil
|
||||
}
|
||||
|
||||
resetTestOrbitClient := func() {
|
||||
getInstallerDetailsFnCalled = false
|
||||
installIdRequested = ""
|
||||
oc.getInstallerDetailsFn = getInstallerDetailsDefaultFn
|
||||
installDetails = &fleet.SoftwareInstallDetails{
|
||||
ExecutionID: "exec1",
|
||||
InstallerID: 1337,
|
||||
InstallScript: "script1",
|
||||
PostInstallScript: "script2",
|
||||
SoftwareInstallerURL: &fleet.SoftwareInstallerURL{
|
||||
URL: "https://example.com/ABC",
|
||||
Filename: "installer.pkg",
|
||||
},
|
||||
}
|
||||
downloadInstallerFnCalled = false
|
||||
downloadInstallerFromURLFnCalled = false
|
||||
oc.downloadInstallerFromURLFn = downloadInstallerFromURLDefaultFn
|
||||
savedInstallerResult = nil
|
||||
}
|
||||
|
||||
r := &Runner{
|
||||
OrbitClient: oc,
|
||||
scriptsEnabled: func() bool { return true },
|
||||
}
|
||||
|
||||
var execCalled bool
|
||||
var executedScripts []string
|
||||
var execEnv []string
|
||||
var execErr error
|
||||
execOutput := []byte("execOutput")
|
||||
execExitCode := 0
|
||||
execCmdDefaultFn := func(ctx context.Context, scriptPath string, env []string) ([]byte, int, error) {
|
||||
execCalled = true
|
||||
execEnv = env
|
||||
executedScripts = append(executedScripts, scriptPath)
|
||||
return execOutput, execExitCode, execErr
|
||||
}
|
||||
r.execCmdFn = execCmdDefaultFn
|
||||
|
||||
var tmpDirFnCalled bool
|
||||
var tmpDir string
|
||||
r.tempDirFn = func(dir, pattern string) (string, error) {
|
||||
tmpDirFnCalled = true
|
||||
tmpDir = os.TempDir()
|
||||
return tmpDir, nil
|
||||
}
|
||||
|
||||
var removeAllFnCalled bool
|
||||
var removedDir string
|
||||
r.removeAllFn = func(s string) error {
|
||||
removedDir = s
|
||||
removeAllFnCalled = true
|
||||
return nil
|
||||
}
|
||||
|
||||
resetRunner := func() {
|
||||
execCalled = false
|
||||
executedScripts = nil
|
||||
execEnv = nil
|
||||
execOutput = []byte("execOutput")
|
||||
execExitCode = 0
|
||||
execErr = nil
|
||||
r.execCmdFn = execCmdDefaultFn
|
||||
|
||||
tmpDirFnCalled = false
|
||||
tmpDir = ""
|
||||
}
|
||||
|
||||
var config fleet.OrbitConfig
|
||||
config.Notifications.PendingSoftwareInstallerIDs = []string{installDetails.ExecutionID}
|
||||
|
||||
resetConfig := func() {
|
||||
config.Notifications.PendingSoftwareInstallerIDs = []string{installDetails.ExecutionID}
|
||||
}
|
||||
|
||||
resetAll := func() {
|
||||
resetTestOrbitClient()
|
||||
resetRunner()
|
||||
resetConfig()
|
||||
}
|
||||
|
||||
t.Run("everything good", func(t *testing.T) {
|
||||
resetAll()
|
||||
|
||||
err := r.run(context.Background(), &config)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, removeAllFnCalled)
|
||||
assert.Equal(t, tmpDir, removedDir)
|
||||
|
||||
assert.True(t, tmpDirFnCalled)
|
||||
|
||||
assert.True(t, execCalled)
|
||||
scriptExtension := ".sh"
|
||||
if runtime.GOOS == "windows" {
|
||||
scriptExtension = ".ps1"
|
||||
}
|
||||
assert.Contains(t, executedScripts, filepath.Join(tmpDir, "install-script"+scriptExtension))
|
||||
assert.Contains(t, executedScripts, filepath.Join(tmpDir, "post-install-script"+scriptExtension))
|
||||
assert.Contains(t, execEnv, "INSTALLER_PATH="+filepath.Join(tmpDir, installDetails.SoftwareInstallerURL.Filename))
|
||||
|
||||
assert.NotNil(t, savedInstallerResult)
|
||||
assert.Equal(t, execExitCode, *savedInstallerResult.InstallScriptExitCode)
|
||||
assert.Equal(t, string(execOutput), *savedInstallerResult.InstallScriptOutput)
|
||||
assert.Equal(t, execExitCode, *savedInstallerResult.PostInstallScriptExitCode)
|
||||
assert.Equal(t, string(execOutput), *savedInstallerResult.PostInstallScriptOutput)
|
||||
assert.Equal(t, installDetails.ExecutionID, savedInstallerResult.InstallUUID)
|
||||
|
||||
assert.True(t, downloadInstallerFromURLFnCalled)
|
||||
assert.False(t, downloadInstallerFnCalled)
|
||||
|
||||
assert.True(t, getInstallerDetailsFnCalled)
|
||||
assert.Equal(t, installDetails.ExecutionID, installIdRequested)
|
||||
})
|
||||
|
||||
t.Run("CDN fails and we fall back to Fleet download", func(t *testing.T) {
|
||||
resetAll()
|
||||
|
||||
oc.downloadInstallerFromURLFn = func(url string, filename string, downloadDir string) (string, error) {
|
||||
assert.Equal(t, installDetails.SoftwareInstallerURL.URL, url)
|
||||
downloadInstallerFromURLFnCalled = true
|
||||
return "bozo", errors.New("test error")
|
||||
}
|
||||
|
||||
err := r.run(context.Background(), &config)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, removeAllFnCalled)
|
||||
assert.Equal(t, tmpDir, removedDir)
|
||||
|
||||
assert.True(t, tmpDirFnCalled)
|
||||
|
||||
assert.True(t, execCalled)
|
||||
scriptExtension := ".sh"
|
||||
if runtime.GOOS == "windows" {
|
||||
scriptExtension = ".ps1"
|
||||
}
|
||||
assert.Contains(t, executedScripts, filepath.Join(tmpDir, "install-script"+scriptExtension))
|
||||
assert.Contains(t, executedScripts, filepath.Join(tmpDir, "post-install-script"+scriptExtension))
|
||||
require.Contains(t, execEnv, "INSTALLER_PATH="+filepath.Join(tmpDir, fmt.Sprint(installDetails.InstallerID)+".pkg"))
|
||||
|
||||
assert.NotNil(t, savedInstallerResult)
|
||||
assert.Equal(t, execExitCode, *savedInstallerResult.InstallScriptExitCode)
|
||||
assert.Equal(t, string(execOutput), *savedInstallerResult.InstallScriptOutput)
|
||||
assert.Equal(t, execExitCode, *savedInstallerResult.PostInstallScriptExitCode)
|
||||
assert.Equal(t, string(execOutput), *savedInstallerResult.PostInstallScriptOutput)
|
||||
assert.Equal(t, installDetails.ExecutionID, savedInstallerResult.InstallUUID)
|
||||
|
||||
assert.True(t, downloadInstallerFromURLFnCalled)
|
||||
assert.True(t, downloadInstallerFnCalled)
|
||||
|
||||
assert.True(t, getInstallerDetailsFnCalled)
|
||||
assert.Equal(t, installDetails.ExecutionID, installIdRequested)
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
func TestScriptsDisabled(t *testing.T) {
|
||||
oc := &TestOrbitClient{}
|
||||
qc := &TestQueryClient{}
|
||||
|
||||
@@ -196,18 +196,25 @@ type bodyHandler interface {
|
||||
}
|
||||
|
||||
type FileResponse struct {
|
||||
DestPath string
|
||||
DestFile string
|
||||
destFilePath string
|
||||
DestPath string
|
||||
DestFile string
|
||||
destFilePath string
|
||||
SkipMediaType bool
|
||||
}
|
||||
|
||||
func (f *FileResponse) Handle(resp *http.Response) error {
|
||||
_, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing media type from response header: %w", err)
|
||||
var filename string
|
||||
if !f.SkipMediaType {
|
||||
_, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing media type from response header: %w", err)
|
||||
}
|
||||
filename = params["filename"]
|
||||
}
|
||||
|
||||
filename := params["filename"]
|
||||
if filename == "" {
|
||||
filename = f.DestFile
|
||||
}
|
||||
if filename == "" {
|
||||
filename = uuid.NewString()
|
||||
}
|
||||
|
||||
@@ -1138,7 +1138,8 @@ func (svc *Service) validateAndEncrypt(ctx context.Context, passphrase string, s
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
type orbitGetSoftwareInstallRequest struct {
|
||||
OrbitNodeKey string `json:"orbot_node_key"`
|
||||
OrbitNodeKey string `json:"orbit_node_key"`
|
||||
OrbotNodeKey string `json:"orbot_node_key"` // legacy typo -- keep for backwards compatibility with orbit <= 1.38.0
|
||||
InstallUUID string `json:"install_uuid"`
|
||||
}
|
||||
|
||||
@@ -1149,7 +1150,10 @@ func (r *orbitGetSoftwareInstallRequest) setOrbitNodeKey(nodeKey string) {
|
||||
|
||||
// interface implementation required by the OrbitClient
|
||||
func (r *orbitGetSoftwareInstallRequest) orbitHostNodeKey() string {
|
||||
return r.OrbitNodeKey
|
||||
if r.OrbitNodeKey != "" {
|
||||
return r.OrbitNodeKey
|
||||
}
|
||||
return r.OrbotNodeKey
|
||||
}
|
||||
|
||||
type orbitGetSoftwareInstallResponse struct {
|
||||
|
||||
@@ -73,6 +73,12 @@ type configCache struct {
|
||||
}
|
||||
|
||||
func (oc *OrbitClient) request(verb string, path string, params interface{}, resp interface{}) error {
|
||||
return oc.requestWithExternal(verb, path, params, resp, false)
|
||||
}
|
||||
|
||||
// requestWithExternal is used to make requests to Fleet or external URLs. If external is true, the pathOrURL
|
||||
// is used as the full URL to make the request to.
|
||||
func (oc *OrbitClient) requestWithExternal(verb string, pathOrURL string, params interface{}, resp interface{}, external bool) error {
|
||||
var bodyBytes []byte
|
||||
var err error
|
||||
if params != nil {
|
||||
@@ -82,11 +88,6 @@ func (oc *OrbitClient) request(verb string, path string, params interface{}, res
|
||||
}
|
||||
}
|
||||
|
||||
parsedURL, err := url.Parse(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing URL: %w", err)
|
||||
}
|
||||
|
||||
oc.closeIdleConnections()
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -94,24 +95,42 @@ func (oc *OrbitClient) request(verb string, path string, params interface{}, res
|
||||
ctx = httptrace.WithClientTrace(ctx, testStdoutHTTPTracer)
|
||||
}
|
||||
|
||||
request, err := http.NewRequestWithContext(
|
||||
ctx,
|
||||
verb,
|
||||
oc.url(parsedURL.Path, parsedURL.RawQuery).String(),
|
||||
bytes.NewBuffer(bodyBytes),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
var request *http.Request
|
||||
if external {
|
||||
request, err = http.NewRequestWithContext(
|
||||
ctx,
|
||||
verb,
|
||||
pathOrURL,
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
parsedURL, err := url.Parse(pathOrURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing URL: %w", err)
|
||||
}
|
||||
|
||||
request, err = http.NewRequestWithContext(
|
||||
ctx,
|
||||
verb,
|
||||
oc.url(parsedURL.Path, parsedURL.RawQuery).String(),
|
||||
bytes.NewBuffer(bodyBytes),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
oc.setClientCapabilitiesHeader(request)
|
||||
}
|
||||
oc.setClientCapabilitiesHeader(request)
|
||||
response, err := oc.http.Do(request)
|
||||
if err != nil {
|
||||
oc.setLastRecordedError(err)
|
||||
return fmt.Errorf("%s %s: %w", verb, path, err)
|
||||
return fmt.Errorf("%s %s: %w", verb, pathOrURL, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
if err := oc.parseResponse(verb, path, response, resp); err != nil {
|
||||
if err := oc.parseResponse(verb, pathOrURL, response, resp); err != nil {
|
||||
oc.setLastRecordedError(err)
|
||||
return err
|
||||
}
|
||||
@@ -413,6 +432,14 @@ func (oc *OrbitClient) DownloadSoftwareInstaller(installerID uint, downloadDirec
|
||||
return resp.GetFilePath(), nil
|
||||
}
|
||||
|
||||
func (oc *OrbitClient) DownloadSoftwareInstallerFromURL(url string, filename string, downloadDirectory string) (string, error) {
|
||||
resp := FileResponse{DestPath: downloadDirectory, DestFile: filename, SkipMediaType: true}
|
||||
if err := oc.requestWithExternal("GET", url, nil, &resp, true); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return resp.GetFilePath(), nil
|
||||
}
|
||||
|
||||
type NullFileResponse struct{}
|
||||
|
||||
func (f *NullFileResponse) Handle(resp *http.Response) error {
|
||||
|
||||
Reference in New Issue
Block a user