Add software installer upload/download progress to GitOps runs (#50250)

<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #45728
Changes:
- Adds a new redis key to keep track of downloaded packages. It starts
out with an empty list and gets filled with each download. Each update
writes the entire struct at once to the key.
- Adds logging in the fleetctl gitops client to show which packages were
downloaded
- Fixes the categories key potentially expiring 

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

- [ ] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements), JS
inline code is prevented especially for url redirects, and untrusted
data interpolated into shell scripts/commands is validated against shell
metacharacters.
- ❌ Timeouts are implemented and retries are limited to avoid infinite
loops
- Right now the batch will write the whole slice of all packages to a
single redis key for every package in the loop. Looks like performance
is acceptable for now (500 packages), but maybe this will need to be
limited.
- [ ] If paths of existing endpoints are modified without backwards
compatibility, checked the frontend/CLI for any necessary changes

## Testing

- [x] Added/updated automated tests
- [ ] Where appropriate, [automated tests simulate multiple hosts and
test for host
isolation](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/reference/patterns-backend.md#unit-testing)
(updates to one hosts's records do not affect another)

- [x] QA'd all new/changed functionality manually



<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## New Features
- Added per-package software download progress in fleetctl GitOps.
- Progress now reports downloading, completed, skipped, and failed
packages during real and dry runs.
- Installation output now distinguishes applying and applied stages.

## Bug Fixes
- Improved download error messages and cached-package handling.
- Prevented duplicate progress messages and ensured tracking issues do
not interrupt successful software batches.

## Tests
- Expanded coverage for progress reporting, failures, dry runs, package
types, and authorization scenarios.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

cmd/fleetctl/integrationtest/gitops/gitops_enterprise_integration_test.go

@@ -228,7 +228,9 @@ func (s *enterpriseIntegrationGitopsTestSuite) assertDryRunOutputWithDeprecation
 		"created",
 		"set",
 	}
-	pattern := fmt.Sprintf("\\[([+\\-!])] would've (%s)", strings.Join(allowedVerbs, "|"))
+	// A dry run downloads software packages in full, so it reports each download as it
+	// happens. Those lines say what it did, not what it would do.
+	pattern := fmt.Sprintf("\\[([+\\-!])] (would've (%s)|downloading|downloaded|skipped)", strings.Join(allowedVerbs, "|"))
 	reg := regexp.MustCompile(pattern)
 	for line := range strings.SplitSeq(output, "\n") {
 		if expectDeprecation && line != "" && strings.Contains(line, "is deprecated") {
@@ -257,8 +259,11 @@ func (s *enterpriseIntegrationGitopsTestSuite) assertRealRunOutputWithDeprecatio
 		"added",
 		"created",
 		"set",
-		"applying", // this is used when doing groups operations before the operation starts, e.g. "Applying 10 policies"
-		"deleting", // ditto
+		"applying",    // this is used when doing groups operations before the operation starts, e.g. "Applying 10 policies"
+		"deleting",    // ditto
+		"downloading", // software packages report each download as it starts
+		"downloaded",  // ditto, as it finishes
+		"skipped",     // ditto, for a package already in storage
 	}
 	pattern := fmt.Sprintf("\\[([+\\-!])] (%s)", strings.Join(allowedVerbs, "|"))
 	reg := regexp.MustCompile(pattern)
@@ -4520,7 +4525,7 @@ team_settings:
 			teamName:     teamName,
 			teamTemplate: testPackages,
 			teamSettings: `secrets: [{"secret":"enroll_secret"}]`,
-			errContains:  ptr.String("Couldn't edit software."),
+			errContains:  new(`"setup_experience" cannot be used for macOS software if "macos_manual_agent_install" is enabled.`),
 		},
 		{
 			testName:     "No team VPP",
@@ -4534,7 +4539,7 @@ team_settings:
 			VPPTeam:      "No team",
 			teamName:     "Unassigned",
 			teamTemplate: testPackages,
-			errContains:  ptr.String("Couldn't edit software."),
+			errContains:  new(`"setup_experience" cannot be used for macOS software if "macos_manual_agent_install" is enabled.`),
 		},
 		// left out more possible combinations of setup experience being set for different platforms
 	}

cmd/fleetctl/integrationtest/gitops/software_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/fleetdm/fleet/v4/cmd/fleetctl/fleetctl/fleetctltest"
 	"github.com/fleetdm/fleet/v4/cmd/fleetctl/fleetctl/testing_utils"
 	"github.com/fleetdm/fleet/v4/pkg/file"
+	"github.com/fleetdm/fleet/v4/server/dev_mode"
 	"github.com/fleetdm/fleet/v4/server/fleet"
 	"github.com/fleetdm/fleet/v4/server/ptr"
 	"github.com/fleetdm/fleet/v4/server/test"
@@ -1608,3 +1609,72 @@ func TestGitOpsTeamInHouseAppleConfiguration(t *testing.T) {
 		})
 	}
 }
+
+func TestGitOpsSoftwareDownloadProgress(t *testing.T) {
+	testing_utils.StartSoftwareInstallerServer(t)
+	dev_mode.SetOverride("FLEET_DEV_BATCH_RETRY_INTERVAL", "1s")
+	t.Cleanup(func() { dev_mode.ClearOverride("FLEET_DEV_BATCH_RETRY_INTERVAL") })
+
+	file := "../../fleetctl/testdata/gitops/team_software_installer_valid.yml"
+
+	// The batch needs these to get as far as downloading; the harness doesn't set them.
+	setupSoftwareMocks := func(t *testing.T) map[string]**fleet.Team {
+		ds, _, savedTeams := testing_utils.SetupFullGitOpsPremiumServer(t)
+		ds.GetTeamsWithInstallerByHashFunc = func(ctx context.Context, sha256, url string) (map[uint][]*fleet.ExistingSoftwareInstaller, error) {
+			return map[uint][]*fleet.ExistingSoftwareInstaller{}, nil
+		}
+		ds.GetInstallerByTeamAndURLFunc = func(ctx context.Context, teamID *uint, url string) (*fleet.ExistingSoftwareInstaller, error) {
+			return nil, nil
+		}
+		ds.GetSoftwareCategoryNameToIDMapFunc = func(ctx context.Context, teamID uint, names []string) (map[string]uint, error) {
+			return map[string]uint{}, nil
+		}
+		return savedTeams
+	}
+
+	t.Run("a package Fleet downloads reports its progress", func(t *testing.T) {
+		setupSoftwareMocks(t)
+
+		out, err := fleetctltest.RunAppNoChecks([]string{"gitops", "-f", file})
+		require.NoError(t, err)
+		require.Contains(t, out.String(), "[+] applying 2 software packages for fleet "+teamName+"\n")
+		require.Contains(t, out.String(), "[+] downloading software package - ruby.deb ...\n")
+		require.Contains(t, out.String(), "[+] downloaded software package - ruby.deb\n")
+		require.Contains(t, out.String(), "[+] applied 2 software packages for fleet "+teamName+"\n")
+	})
+
+	t.Run("a dry run for an existing fleet reports the same progress", func(t *testing.T) {
+		// A dry run for a fleet that doesn't exist yet never starts a batch, so there is
+		// nothing to download and nothing to report.
+		savedTeams := setupSoftwareMocks(t)
+		team := &fleet.Team{ID: 1, Name: teamName}
+		savedTeams[teamName] = &team
+
+		out, err := fleetctltest.RunAppNoChecks([]string{"gitops", "--dry-run", "-f", file})
+		require.NoError(t, err)
+		require.Contains(t, out.String(), "[+] downloading software package - ruby.deb ...\n")
+		require.Contains(t, out.String(), "[+] downloaded software package - ruby.deb\n")
+		require.Contains(t, out.String(), "[+] would've applied 2 software packages for fleet "+teamName+"\n")
+	})
+
+	t.Run("a script package stays out of the progress, since nothing is downloaded for it", func(t *testing.T) {
+		setupSoftwareMocks(t)
+
+		out, err := fleetctltest.RunAppNoChecks([]string{"gitops", "-f", "../../fleetctl/testdata/gitops/team_software_script_package.yml"})
+		require.NoError(t, err)
+		// The counts prove the script package was in the batch, not just missing from it.
+		require.Contains(t, out.String(), "[+] applying 2 software packages for fleet "+teamName+"\n")
+		require.Contains(t, out.String(), "[+] downloaded software package - ruby.deb\n")
+		require.NotContains(t, out.String(), "install_ruby.sh")
+		require.Contains(t, out.String(), "[+] applied 2 software packages for fleet "+teamName+"\n")
+	})
+
+	t.Run("a package Fleet can't download reports the failure", func(t *testing.T) {
+		setupSoftwareMocks(t)
+
+		out, err := fleetctltest.RunAppNoChecks([]string{"gitops", "-f", "../../fleetctl/testdata/gitops/team_software_installer_not_found.yml"})
+		require.Error(t, err)
+		require.Contains(t, out.String(), "Error: could not download software package notfound.deb\n")
+		require.NotContains(t, out.String(), "[+] downloaded software package - notfound.deb")
+	})
+}