Kickstart sofwareupdated periodically from fleetd/orbit to work around a macOS bug (#9465)

orbit/changes/issue-9347-kickstart-softwareupdated

@@ -0,0 +1 @@
+* Added periodical restart of the `softwareupdated` service to work around a macOS bug where it sometimes hangs and prevents software updates.

orbit/cmd/orbit/orbit.go

@@ -139,6 +139,12 @@ func main() {
 			Usage:   "Launch Fleet Desktop application (flag currently only used on darwin)",
 			EnvVars: []string{"ORBIT_FLEET_DESKTOP"},
 		},
+		&cli.BoolFlag{
+			Name:    "disable-kickstart-softwareupdated",
+			Usage:   "Disable periodic execution of 'launchctl kickstart -k softwareupdated' on macOS",
+			EnvVars: []string{"ORBIT_FLEET_DISABLE_KICKSTART_SOFTWAREUPDATED"},
+			Hidden:  true,
+		},
 	}
 	app.Before = func(c *cli.Context) error {
 		// handle old installations, which had default root dir set to /var/lib/orbit
@@ -267,6 +273,15 @@ func main() {
 		g.Add(systemChecker.Execute, systemChecker.Interrupt)
 		go osservice.SetupServiceManagement(constant.SystemServiceName, systemChecker.svcInterruptCh, appDoneCh)
 
+		// periodically run launchctl kickstart -k softwareupdated on macOS
+		if runtime.GOOS == "darwin" && !c.Bool("disable-kickstart-softwareupdated") {
+			const softwareUpdatedKickstartInterval = 12 * time.Hour
+			updatedRunner := update.NewSoftwareUpdatedRunner(update.SoftwareUpdatedOptions{
+				Interval: softwareUpdatedKickstartInterval,
+			})
+			g.Add(updatedRunner.Execute, updatedRunner.Interrupt)
+		}
+
 		// NOTE: When running in dev-mode, even if `disable-updates` is set,
 		// it fetches osqueryd once as part of initialization.
 		var updater *update.Updater

orbit/pkg/update/execcmd.go

@@ -1,5 +1,3 @@
-//go:build darwin
-
 package update
 
 import (
@@ -7,8 +5,10 @@ import (
 	"os/exec"
 )
 
-func runRenewEnrollmentProfile() error {
-	cmd := exec.Command("/usr/bin/profiles", "renew", "--type", "enrollment")
+var _ = runCmdCollectErr // just to avoid unused errors on non-darwin platforms
+
+func runCmdCollectErr(exe string, args ...string) error {
+	cmd := exec.Command(exe, args...)
 	out, err := cmd.CombinedOutput()
 	if err != nil && len(out) > 0 {
 		// just as a precaution, limit the length of the output

orbit/pkg/update/execcmd_darwin.go

@@ -0,0 +1,11 @@
+//go:build darwin
+
+package update
+
+func runRenewEnrollmentProfile() error {
+	return runCmdCollectErr("/usr/bin/profiles", "renew", "--type", "enrollment")
+}
+
+func runKickstartSoftwareUpdated() error {
+	return runCmdCollectErr("launchctl", "kickstart", "-k", "system/com.apple.softwareupdated")
+}

orbit/pkg/update/execcmd_stub.go

@@ -5,3 +5,7 @@ package update
 func runRenewEnrollmentProfile() error {
 	return nil
 }
+
+func runKickstartSoftwareUpdated() error {
+	return nil
+}

orbit/pkg/update/softwareupdated_runner.go

@@ -0,0 +1,79 @@
+package update
+
+import (
+	"time"
+
+	"github.com/rs/zerolog/log"
+)
+
+// SoftwareUpdatedRunner is a specialized runner to periodically kickstart the
+// softwareupdated service on macOS, to work around a bug where the service
+// hangs from time to time and prevents updates from being downloaded or update
+// notifications from being shown.
+//
+// It is designed with Execute and Interrupt functions to be compatible with
+// oklog/run.
+type SoftwareUpdatedRunner struct {
+	opt    SoftwareUpdatedOptions
+	cancel chan struct{}
+}
+
+// SoftwareUpdatedOptions defines the options provided for the softwareupdated
+// runner.
+type SoftwareUpdatedOptions struct {
+	// Interval is the interval at which to run the the kickstart softwareupdated
+	// command.
+	Interval time.Duration
+
+	// runCmdFn can be set in tests to mock the command executed to kickstart
+	// softwareupdated. If nil, defaults to runKickstartSoftwareUpdated.
+	runCmdFn runCmdFunc
+}
+
+// NewSoftwareUpdatedRunner creates a new runner with the provided options. The
+// runner must be started with Execute.
+func NewSoftwareUpdatedRunner(opt SoftwareUpdatedOptions) *SoftwareUpdatedRunner {
+	return &SoftwareUpdatedRunner{
+		opt:    opt,
+		cancel: make(chan struct{}),
+	}
+}
+
+// Execute starts the loop to periodically run the kickstart command.
+func (r *SoftwareUpdatedRunner) Execute() error {
+	log.Debug().Msg("starting softwareupdated runner")
+
+	// ensure it runs ~immediately the first time (e.g. on startup)
+	firstInterval := 10 * time.Second
+	if r.opt.Interval < firstInterval {
+		firstInterval = r.opt.Interval
+	}
+	ticker := time.NewTicker(firstInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-r.cancel:
+			return nil
+
+		case <-ticker.C:
+			log.Info().Msg("executing launchctl kickstart -k softwareupdated")
+			fn := r.opt.runCmdFn
+			if fn == nil {
+				fn = runKickstartSoftwareUpdated
+			}
+			if err := fn(); err != nil {
+				log.Info().Err(err).Msg("executing launchctl kickstart -k softwareupdated failed")
+			}
+			// run at the defined interval the next time around
+			ticker.Reset(r.opt.Interval)
+		}
+	}
+}
+
+// Interrupt is the oklog/run interrupt method that stops the runner when
+// called.
+func (r *SoftwareUpdatedRunner) Interrupt(err error) {
+	close(r.cancel)
+	log.Debug().Err(err).Msg("interrupt for softwareupdated runner")
+}

orbit/pkg/update/softwareupdated_runner_test.go

@@ -0,0 +1,32 @@
+package update
+
+import (
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/oklog/run"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSoftwareUpdatedRunner(t *testing.T) {
+	var callCount int32
+	runFn := func() error {
+		atomic.AddInt32(&callCount, 1)
+		return nil
+	}
+
+	var g run.Group
+	// add the "lead" runner, it will return after 1s and cause all runners to stop
+	g.Add(func() error { time.Sleep(time.Second); return nil }, func(error) {})
+
+	// add the softwareupdated runner, should run at least 2 times (leave some
+	// wiggle room in case it is so slow on CI that it is not scheduled more
+	// often).
+	r := NewSoftwareUpdatedRunner(SoftwareUpdatedOptions{Interval: 300 * time.Millisecond, runCmdFn: runFn})
+	g.Add(r.Execute, r.Interrupt)
+
+	err := g.Run()
+	require.NoError(t, err)
+	require.GreaterOrEqual(t, atomic.LoadInt32(&callCount), int32(2))
+}