diff --git a/changes/issue-7130-native-notarization-retry b/changes/issue-7130-native-notarization-retry new file mode 100644 index 0000000000..ee64971d6b --- /dev/null +++ b/changes/issue-7130-native-notarization-retry @@ -0,0 +1 @@ +* Added logic to retry notarization and codesigning when using `FLEETCTL_NATIVE_TOOLING` diff --git a/orbit/pkg/packaging/macos_rcodesign.go b/orbit/pkg/packaging/macos_rcodesign.go index 9d8acc2fb2..b65f2a9396 100644 --- a/orbit/pkg/packaging/macos_rcodesign.go +++ b/orbit/pkg/packaging/macos_rcodesign.go @@ -7,6 +7,7 @@ import ( "os/exec" "path/filepath" + "github.com/fleetdm/fleet/v4/pkg/retry" "github.com/fleetdm/fleet/v4/pkg/secure" ) @@ -18,20 +19,22 @@ func rSign(pkgPath, cert string) error { return fmt.Errorf("writing cert data: %e", err) } - var outBuf bytes.Buffer - cmd := exec.Command( - "rcodesign", - "sign", - pkgPath, - "--pem-source", pemPath, - ) - cmd.Stdout = &outBuf - cmd.Stderr = &outBuf - if err := cmd.Run(); err != nil { - fmt.Println(outBuf.String()) - return fmt.Errorf("rcodesign: %w", err) - } - return nil + return retry.Do(func() error { + var outBuf bytes.Buffer + cmd := exec.Command( + "rcodesign", + "sign", + pkgPath, + "--pem-source", pemPath, + ) + cmd.Stdout = &outBuf + cmd.Stderr = &outBuf + if err := cmd.Run(); err != nil { + fmt.Println(outBuf.String()) + return fmt.Errorf("rcodesign: %w", err) + } + return nil + }, retry.WithMaxAttempts(3)) } func rNotarizeStaple(pkg, apiKeyID, apiKeyIssuer, apiKeyContent string) error { @@ -40,21 +43,24 @@ func rNotarizeStaple(pkg, apiKeyID, apiKeyIssuer, apiKeyContent string) error { if err != nil { return fmt.Errorf("writing API keys: %e", err) } - var outBuf bytes.Buffer - cmd := exec.Command("rcodesign", - "notarize", - pkg, - "--api-issuer", apiKeyIssuer, - "--api-key", apiKeyID, - "--staple", - ) - cmd.Stdout = &outBuf - cmd.Stderr = &outBuf - if err := cmd.Run(); err != nil { - fmt.Println(outBuf.String()) - return fmt.Errorf("rcodesign notarize: %w", err) - } - return nil + + return retry.Do(func() error { + var outBuf bytes.Buffer + cmd := exec.Command("rcodesign", + "notarize", + pkg, + "--api-issuer", apiKeyIssuer, + "--api-key", apiKeyID, + "--staple", + ) + cmd.Stdout = &outBuf + cmd.Stderr = &outBuf + if err := cmd.Run(); err != nil { + fmt.Println(outBuf.String()) + return fmt.Errorf("rcodesign notarize: %w", err) + } + return nil + }, retry.WithMaxAttempts(3)) } func writeAPIKeys(issuer, id, content string) (string, error) { diff --git a/pkg/nettest/nettest.go b/pkg/nettest/nettest.go index 00c6dc46b7..2f46bb0ba1 100644 --- a/pkg/nettest/nettest.go +++ b/pkg/nettest/nettest.go @@ -9,6 +9,8 @@ import ( "strings" "testing" "time" + + "github.com/fleetdm/fleet/v4/pkg/retry" ) func lock(lockFilePath string) { @@ -74,13 +76,10 @@ func Retryable(err error) bool { // RunWithNetRetry runs the given function and retries in case of network errors (see Retryable). func RunWithNetRetry(t *testing.T, fn func() error) error { - for { - err := fn() - if err != nil && Retryable(err) { - time.Sleep(5 * time.Second) + return retry.Do(func() error { + if err := fn(); err != nil && Retryable(err) { t.Logf("%s: retrying error: %s", t.Name(), err) - continue } - return err - } + return nil + }, retry.WithInterval(5*time.Second)) } diff --git a/pkg/retry/retry.go b/pkg/retry/retry.go new file mode 100644 index 0000000000..6121a2b258 --- /dev/null +++ b/pkg/retry/retry.go @@ -0,0 +1,63 @@ +// package retry has utilities to retry operations +package retry + +import ( + "time" +) + +type config struct { + interval time.Duration + maxAttempts int +} + +// Option allows to configure the behavior of retry.Do +type Option func(*config) + +// WithRetryInterval allows to specify a custom duration to wait +// between retries. +func WithInterval(i time.Duration) Option { + return func(c *config) { + c.interval = i + } +} + +// WithMaxAttempts allows to specify a maximum number of attempts +// before the doer gives up +func WithMaxAttempts(a int) Option { + return func(c *config) { + c.maxAttempts = a + } +} + +// Do executes the provided function, if the function returns a +// non-nil error it performs a retry according to the options +// provided. +// +// By default operations are retried an unlimited number of times for 30 +// seconds +func Do(fn func() error, opts ...Option) error { + cfg := &config{ + interval: 30 * time.Second, + } + for _, opt := range opts { + opt(cfg) + } + + attempts := 0 + ticker := time.NewTicker(cfg.interval) + defer ticker.Stop() + + for { + attempts++ + err := fn() + if err == nil { + return nil + } + + if cfg.maxAttempts != 0 && attempts >= cfg.maxAttempts { + return err + } + + <-ticker.C + } +} diff --git a/pkg/retry/retry_test.go b/pkg/retry/retry_test.go new file mode 100644 index 0000000000..7907778ce8 --- /dev/null +++ b/pkg/retry/retry_test.go @@ -0,0 +1,41 @@ +package retry + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +var errTest = errors.New("test error") + +func TestRetryDo(t *testing.T) { + t.Run("WithMaxAttempts only performs the operation the configured number of times", func(t *testing.T) { + count := 0 + max := 3 + + err := Do(func() error { + count++ + return errTest + }, WithMaxAttempts(max), WithInterval(1*time.Millisecond)) + + require.ErrorIs(t, errTest, err) + require.Equal(t, max, count) + }) + + t.Run("operations are run an unlimited number of times by default", func(t *testing.T) { + count := 0 + max := 10 + + err := Do(func() error { + if count++; count != max { + return errTest + } + return nil + }, WithInterval(1*time.Millisecond)) + + require.NoError(t, err) + require.Equal(t, max, count) + }) +}