add retry logic for native notarization and codesigning (#7806)

Related to #7130, this adds logic to retry native notarization up to three times if it fails for some reason.

Since we're adding retries in various places, I added a new package under pkg for this purpose.

changes/issue-7130-native-notarization-retry

@@ -0,0 +1 @@
+* Added logic to retry notarization and codesigning when using `FLEETCTL_NATIVE_TOOLING`

orbit/pkg/packaging/macos_rcodesign.go

@@ -7,6 +7,7 @@ import (
 	"os/exec"
 	"path/filepath"
 
+	"github.com/fleetdm/fleet/v4/pkg/retry"
 	"github.com/fleetdm/fleet/v4/pkg/secure"
 )
 
@@ -18,20 +19,22 @@ func rSign(pkgPath, cert string) error {
 		return fmt.Errorf("writing cert data: %e", err)
 	}
 
-	var outBuf bytes.Buffer
-	cmd := exec.Command(
-		"rcodesign",
-		"sign",
-		pkgPath,
-		"--pem-source", pemPath,
-	)
-	cmd.Stdout = &outBuf
-	cmd.Stderr = &outBuf
-	if err := cmd.Run(); err != nil {
-		fmt.Println(outBuf.String())
-		return fmt.Errorf("rcodesign: %w", err)
-	}
-	return nil
+	return retry.Do(func() error {
+		var outBuf bytes.Buffer
+		cmd := exec.Command(
+			"rcodesign",
+			"sign",
+			pkgPath,
+			"--pem-source", pemPath,
+		)
+		cmd.Stdout = &outBuf
+		cmd.Stderr = &outBuf
+		if err := cmd.Run(); err != nil {
+			fmt.Println(outBuf.String())
+			return fmt.Errorf("rcodesign: %w", err)
+		}
+		return nil
+	}, retry.WithMaxAttempts(3))
 }
 
 func rNotarizeStaple(pkg, apiKeyID, apiKeyIssuer, apiKeyContent string) error {
@@ -40,21 +43,24 @@ func rNotarizeStaple(pkg, apiKeyID, apiKeyIssuer, apiKeyContent string) error {
 	if err != nil {
 		return fmt.Errorf("writing API keys: %e", err)
 	}
-	var outBuf bytes.Buffer
-	cmd := exec.Command("rcodesign",
-		"notarize",
-		pkg,
-		"--api-issuer", apiKeyIssuer,
-		"--api-key", apiKeyID,
-		"--staple",
-	)
-	cmd.Stdout = &outBuf
-	cmd.Stderr = &outBuf
-	if err := cmd.Run(); err != nil {
-		fmt.Println(outBuf.String())
-		return fmt.Errorf("rcodesign notarize: %w", err)
-	}
-	return nil
+
+	return retry.Do(func() error {
+		var outBuf bytes.Buffer
+		cmd := exec.Command("rcodesign",
+			"notarize",
+			pkg,
+			"--api-issuer", apiKeyIssuer,
+			"--api-key", apiKeyID,
+			"--staple",
+		)
+		cmd.Stdout = &outBuf
+		cmd.Stderr = &outBuf
+		if err := cmd.Run(); err != nil {
+			fmt.Println(outBuf.String())
+			return fmt.Errorf("rcodesign notarize: %w", err)
+		}
+		return nil
+	}, retry.WithMaxAttempts(3))
 }
 
 func writeAPIKeys(issuer, id, content string) (string, error) {

pkg/nettest/nettest.go

@@ -9,6 +9,8 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/fleetdm/fleet/v4/pkg/retry"
 )
 
 func lock(lockFilePath string) {
@@ -74,13 +76,10 @@ func Retryable(err error) bool {
 
 // RunWithNetRetry runs the given function and retries in case of network errors (see Retryable).
 func RunWithNetRetry(t *testing.T, fn func() error) error {
-	for {
-		err := fn()
-		if err != nil && Retryable(err) {
-			time.Sleep(5 * time.Second)
+	return retry.Do(func() error {
+		if err := fn(); err != nil && Retryable(err) {
 			t.Logf("%s: retrying error: %s", t.Name(), err)
-			continue
 		}
-		return err
-	}
+		return nil
+	}, retry.WithInterval(5*time.Second))
 }

pkg/retry/retry.go

@@ -0,0 +1,63 @@
+// package retry has utilities to retry operations
+package retry
+
+import (
+	"time"
+)
+
+type config struct {
+	interval    time.Duration
+	maxAttempts int
+}
+
+// Option allows to configure the behavior of retry.Do
+type Option func(*config)
+
+// WithRetryInterval allows to specify a custom duration to wait
+// between retries.
+func WithInterval(i time.Duration) Option {
+	return func(c *config) {
+		c.interval = i
+	}
+}
+
+// WithMaxAttempts allows to specify a maximum number of attempts
+// before the doer gives up
+func WithMaxAttempts(a int) Option {
+	return func(c *config) {
+		c.maxAttempts = a
+	}
+}
+
+// Do executes the provided function, if the function returns a
+// non-nil error it performs a retry according to the options
+// provided.
+//
+// By default operations are retried an unlimited number of times for 30
+// seconds
+func Do(fn func() error, opts ...Option) error {
+	cfg := &config{
+		interval: 30 * time.Second,
+	}
+	for _, opt := range opts {
+		opt(cfg)
+	}
+
+	attempts := 0
+	ticker := time.NewTicker(cfg.interval)
+	defer ticker.Stop()
+
+	for {
+		attempts++
+		err := fn()
+		if err == nil {
+			return nil
+		}
+
+		if cfg.maxAttempts != 0 && attempts >= cfg.maxAttempts {
+			return err
+		}
+
+		<-ticker.C
+	}
+}

pkg/retry/retry_test.go

@@ -0,0 +1,41 @@
+package retry
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+var errTest = errors.New("test error")
+
+func TestRetryDo(t *testing.T) {
+	t.Run("WithMaxAttempts only performs the operation the configured number of times", func(t *testing.T) {
+		count := 0
+		max := 3
+
+		err := Do(func() error {
+			count++
+			return errTest
+		}, WithMaxAttempts(max), WithInterval(1*time.Millisecond))
+
+		require.ErrorIs(t, errTest, err)
+		require.Equal(t, max, count)
+	})
+
+	t.Run("operations are run an unlimited number of times by default", func(t *testing.T) {
+		count := 0
+		max := 10
+
+		err := Do(func() error {
+			if count++; count != max {
+				return errTest
+			}
+			return nil
+		}, WithInterval(1*time.Millisecond))
+
+		require.NoError(t, err)
+		require.Equal(t, max, count)
+	})
+}