Change self-service categories GitOps to not require dedicated key (#47439)

<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [ ] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.
    - Not needed

- [ ] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements), JS
inline code is prevented especially for url redirects, and untrusted
data interpolated into shell scripts/commands is validated against shell
metacharacters.
- [ ] Timeouts are implemented and retries are limited to avoid infinite
loops
- [ ] If paths of existing endpoints are modified without backwards
compatibility, checked the frontend/CLI for any necessary changes

## Testing

- [x] Added/updated automated tests
- [ ] Where appropriate, [automated tests simulate multiple hosts and
test for host
isolation](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/reference/patterns-backend.md#unit-testing)
(updates to one hosts's records do not affect another)

- [x] QA'd all new/changed functionality manually


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Batch software installer and app-association endpoints now return the
list of referenced self-service categories.
* Category fields support an “omit when unset” JSON behavior so omitted
vs empty categories are distinguishable.

* **Bug Fixes**
* Improved category validation (trim + case-insensitive dedupe) and
GitOps reconciliation to remove unused categories.

* **Chores**
* GitOps schema simplified: no separate top-level
self_service_categories; categories are defined inline with packages.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

cmd/fleetctl/fleetctl/generate_gitops.go

@@ -94,7 +94,6 @@ type generateGitopsClient interface {
 	ListFleetMaintainedApps(teamID uint) ([]fleet.MaintainedApp, error)
 	GetFleetMaintainedApp(id uint) (*fleet.MaintainedApp, error)
 	GetVPPTokens() ([]*fleet.VPPTokenDB, error)
-	ListSelfServiceCategories(teamID uint) ([]fleet.SoftwareCategory, error)
 }
 
 // Given a struct type and a field name, return the JSON field name.
@@ -2276,19 +2275,6 @@ func (cmd *GenerateGitopsCommand) generateSoftware(filePath string, teamID uint,
 		result["fleet_maintained_apps"] = fmas
 	}
 
-	categories, err := cmd.Client.ListSelfServiceCategories(teamID)
-	if err != nil {
-		fmt.Fprintf(cmd.CLI.App.ErrWriter, "Error getting self-service categories: %s\n", err)
-		return nil, err
-	}
-	if len(categories) > 0 {
-		names := make([]string, 0, len(categories))
-		for _, c := range categories {
-			names = append(names, c.Name)
-		}
-		result["self_service_categories"] = names
-	}
-
 	return result, nil
 }
 

cmd/fleetctl/fleetctl/generate_gitops_test.go

@@ -34,7 +34,6 @@ type MockClient struct {
 	TeamNameOverride string
 	WithoutMDM       bool
 	WithoutVPP       bool
-	Categories       []fleet.SoftwareCategory
 }
 
 func (c *MockClient) GetAppConfig() (*fleet.EnrichedAppConfig, error) {
@@ -346,13 +345,6 @@ func (MockClient) ListFleetMaintainedApps(teamID uint) ([]fleet.MaintainedApp, e
 	}, nil
 }
 
-func (c MockClient) ListSelfServiceCategories(teamID uint) ([]fleet.SoftwareCategory, error) {
-	if teamID == 1 {
-		return c.Categories, nil
-	}
-	return nil, nil
-}
-
 func (MockClient) GetPolicies(teamID *uint) ([]*fleet.Policy, error) {
 	if teamID == nil {
 		return []*fleet.Policy{
@@ -2589,35 +2581,6 @@ func TestGenerateGitopsExportOrgLogos(t *testing.T) {
 	})
 }
 
-func TestGenerateGitopsEmitsSelfServiceCategories(t *testing.T) {
-	configureFMAManifestServer(t)
-	fleetClient := &MockClient{Categories: []fleet.SoftwareCategory{
-		{ID: 10, Name: "🌎 Browsers", TeamID: 1},
-		{ID: 11, Name: "💼 Engineering", TeamID: 1},
-	}}
-	action := createGenerateGitopsAction(fleetClient)
-
-	tempDir := os.TempDir() + "/" + uuid.New().String()
-	t.Cleanup(func() { _ = os.RemoveAll(tempDir) })
-
-	flagSet := flag.NewFlagSet("test", flag.ContinueOnError)
-	flagSet.String("dir", tempDir, "")
-	buf := new(bytes.Buffer)
-	cliContext := cli.NewContext(&cli.App{Name: "test", Usage: "test", Writer: buf, ErrWriter: buf}, flagSet, nil)
-	require.NoError(t, action(cliContext), buf.String())
-
-	teamYAML, err := os.ReadFile(tempDir + "/fleets/team-a-👍.yml")
-	require.NoError(t, err)
-	assert.Contains(t, string(teamYAML), "self_service_categories:")
-	assert.Contains(t, string(teamYAML), "🌎 Browsers")
-	assert.Contains(t, string(teamYAML), "💼 Engineering")
-
-	// The fleet that returns no categories must NOT emit the key.
-	otherYAML, err := os.ReadFile(tempDir + "/fleets/unassigned.yml")
-	require.NoError(t, err)
-	assert.NotContains(t, string(otherYAML), "self_service_categories:")
-}
-
 func TestGeneratePoliciesPatchPolicyOrphanedFromFleetMaintainedApp(t *testing.T) {
 	fleetClient := &MockClient{}
 	appConfig, err := fleetClient.GetAppConfig()

cmd/fleetctl/fleetctl/get.go

@@ -377,7 +377,7 @@ func getTeamSoftwareSpec(client *service.Client, teamID uint) (*fleet.SoftwareSp
 					LabelsIncludeAny:   scopeLabelNames(pkg.LabelsIncludeAny),
 					LabelsExcludeAny:   scopeLabelNames(pkg.LabelsExcludeAny),
 					LabelsIncludeAll:   scopeLabelNames(pkg.LabelsIncludeAll),
-					Categories:         pkg.Categories,
+					Categories:         optjson.SetSlice(pkg.Categories),
 					InstallDuringSetup: setupExperienceValue(setupSoftwareByTitleID, title.ID),
 				})
 				continue
@@ -389,7 +389,7 @@ func getTeamSoftwareSpec(client *service.Client, teamID uint) (*fleet.SoftwareSp
 				LabelsIncludeAny:   scopeLabelNames(pkg.LabelsIncludeAny),
 				LabelsExcludeAny:   scopeLabelNames(pkg.LabelsExcludeAny),
 				LabelsIncludeAll:   scopeLabelNames(pkg.LabelsIncludeAll),
-				Categories:         pkg.Categories,
+				Categories:         optjson.SetSlice(pkg.Categories),
 				InstallDuringSetup: setupExperienceValue(setupSoftwareByTitleID, title.ID),
 			})
 		case detail.AppStoreApp != nil:

cmd/fleetctl/fleetctl/gitops_test.go

@@ -7496,10 +7496,10 @@ func TestGitOpsSelfServiceCategoriesReconcile(t *testing.T) {
 		listedTeamIDs = append(listedTeamIDs, teamID)
 		return slices.Clone(existing), nil
 	}
-	ds.NewSoftwareCategoryFunc = func(ctx context.Context, teamID uint, name string) (*fleet.SoftwareCategory, error) {
-		added = append(added, name)
+	ds.BatchNewSoftwareCategoriesFunc = func(ctx context.Context, teamID uint, names []string) error {
+		added = append(added, names...)
 		actions = append(actions, "add")
-		return &fleet.SoftwareCategory{ID: uint(1000 + len(added)), Name: name, TeamID: teamID}, nil
+		return nil
 	}
 	ds.SoftwareCategoryFunc = func(ctx context.Context, id uint) (*fleet.SoftwareCategory, error) {
 		for _, c := range existing {
@@ -7515,16 +7515,15 @@ func TestGitOpsSelfServiceCategoriesReconcile(t *testing.T) {
 		return nil
 	}
 
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "script.sh"), []byte(`echo "hello"`), 0o600))
+
 	teamYAML := func(body string) []string {
-		f, err := os.CreateTemp(t.TempDir(), "categories-*.yml")
-		require.NoError(t, err)
-		_, err = f.WriteString("name: Test Fleet\nteam_settings:\n  secrets:\n" + body)
-		require.NoError(t, err)
-		require.NoError(t, f.Close())
-		return []string{"gitops", "-f", f.Name()}
+		path := filepath.Join(dir, "team.yml")
+		require.NoError(t, os.WriteFile(path, []byte("name: Test Fleet\nteam_settings:\n  secrets:\n"+body), 0o600))
+		return []string{"gitops", "-f", path}
 	}
 	unassignedYAML := func(body string) []string {
-		dir := t.TempDir()
 		globalPath := filepath.Join(dir, "global.yml")
 		require.NoError(t, os.WriteFile(globalPath, []byte(`
 controls:
@@ -7559,33 +7558,25 @@ software:
 		{ID: 202, Name: "Stale No-team Category", TeamID: 0},
 	}
 
-	// explicit list reconciles: adds new, keeps existing, deletes stale,
-	// and all inserts run before any deletes (cascade safety).
+	// categories derived from software items reconcile: add new, keep existing,
+	// delete stale, and all inserts run before any deletes (cascade safety).
 	reset(teamExisting, fleet.GitOpsExceptions{})
 	_, err := runAppNoChecks(teamYAML(`controls:
 policies:
 software:
-  self_service_categories:
-    - "🌎 Browsers"
-    - "💼 Engineering"
+  packages:
+    - path: ./script.sh
+      self_service: true
+      categories:
+        - "🌎 Browsers"
+        - "💼 Engineering"
 `))
 	require.NoError(t, err)
 	assert.Equal(t, []string{"💼 Engineering"}, added)
 	assert.ElementsMatch(t, []uint{101, 102}, deleted)
 	assert.Equal(t, []string{"add", "del", "del"}, actions, "all inserts must run before any deletes")
 
-	// empty list deletes all categories.
-	reset(teamExisting, fleet.GitOpsExceptions{})
-	_, err = runAppNoChecks(teamYAML(`controls:
-policies:
-software:
-  self_service_categories: []
-`))
-	require.NoError(t, err)
-	assert.Empty(t, added)
-	assert.ElementsMatch(t, []uint{100, 101, 102}, deleted)
-
-	// absent key leaves categories untouched.
+	// software present with no referenced categories deletes all categories.
 	reset(teamExisting, fleet.GitOpsExceptions{})
 	_, err = runAppNoChecks(teamYAML(`controls:
 policies:
@@ -7594,35 +7585,45 @@ software:
 `))
 	require.NoError(t, err)
 	assert.Empty(t, added)
-	assert.Empty(t, deleted)
+	assert.ElementsMatch(t, []uint{100, 101, 102}, deleted)
 
-	// software exception + software omitted skips reconciliation.
-	reset(teamExisting, fleet.GitOpsExceptions{Software: true})
+	// With the `software:` key absent, software is still declaratively managed
+	// (and cleared), so the fleet's categories are pruned too. This is distinct
+	// from omitting an individual item's `categories:` field. The software
+	// exception is covered in TestGitOpsSelfServiceCategoriesSoftwareException.
+	reset(teamExisting, fleet.GitOpsExceptions{})
 	_, err = runAppNoChecks(teamYAML(`controls:
 policies:
 `))
 	require.NoError(t, err)
 	assert.Empty(t, added)
-	assert.Empty(t, deleted)
+	assert.ElementsMatch(t, []uint{100, 101, 102}, deleted)
 
-	// plain and emoji names treated as distinct.
+	// legacy plain names are normalized to their canonical emoji form, so a
+	// plain and emoji reference to the same default category collapse to one.
 	reset(nil, fleet.GitOpsExceptions{})
 	_, err = runAppNoChecks(teamYAML(`controls:
 policies:
 software:
-  self_service_categories:
-    - "Security"
-    - "🔐 Security"
+  packages:
+    - path: ./script.sh
+      self_service: true
+      categories:
+        - "SECURITY"
+        - "🔐 Security"
 `))
 	require.NoError(t, err)
-	assert.ElementsMatch(t, []string{"Security", "🔐 Security"}, added)
+	assert.Equal(t, []string{"🔐 Security"}, added)
 
-	// no-team explicit list reconciles under team_id = 0.
+	// no-team categories reconcile under team_id = 0.
 	reset(noTeamExisting, fleet.GitOpsExceptions{})
 	_, err = runAppNoChecks(unassignedYAML(`software:
-  self_service_categories:
-    - "🌎 Browsers"
-    - "💼 Engineering"
+  packages:
+    - path: ./script.sh
+      self_service: true
+      categories:
+        - "🌎 Browsers"
+        - "💼 Engineering"
 `))
 	require.NoError(t, err)
 	assert.Equal(t, []string{"💼 Engineering"}, added)
@@ -7631,16 +7632,83 @@ software:
 		assert.EqualValues(t, 0, id, "no-team reconcile must list categories on team 0")
 	}
 
-	// no-team empty list deletes all.
+	// no-team software present with no referenced categories deletes all.
 	reset(noTeamExisting, fleet.GitOpsExceptions{})
 	_, err = runAppNoChecks(unassignedYAML(`software:
-  self_service_categories: []
+  packages: []
 `))
 	require.NoError(t, err)
 	assert.Empty(t, added)
 	assert.ElementsMatch(t, []uint{200, 201, 202}, deleted)
 }
 
+func TestSelfServiceCategoriesPruneSkipped(t *testing.T) {
+	license := &fleet.LicenseInfo{Tier: fleet.TierPremium, Expiration: time.Now().Add(24 * time.Hour)}
+
+	_, ds := testing_utils.RunServerWithMockedDS(t, &service.TestServerOpts{License: license, KeyValueStore: testing_utils.NewMemKeyValueStore()})
+	setupEmptyGitOpsMocks(ds)
+	setupDefaultTeamConfigMocks(ds)
+
+	var deleted []uint
+	existing := []fleet.SoftwareCategory{{ID: 100, Name: "🌎 Browsers", TeamID: 1}, {ID: 102, Name: "Stale Category", TeamID: 1}}
+
+	ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) {
+		return &fleet.AppConfig{GitOpsConfig: fleet.GitOpsConfig{Exceptions: fleet.GitOpsExceptions{Software: true}}}, nil
+	}
+	ds.ListTeamsFunc = func(ctx context.Context, _ fleet.TeamFilter, _ fleet.ListOptions) ([]*fleet.Team, error) {
+		return []*fleet.Team{{ID: 1, Name: "Test Fleet"}}, nil
+	}
+	ds.TeamByNameFunc = func(ctx context.Context, name string) (*fleet.Team, error) {
+		if name == "Test Fleet" {
+			return &fleet.Team{ID: 1, Name: "Test Fleet"}, nil
+		}
+		return nil, &notFoundError{}
+	}
+	ds.TeamExistsFunc = func(ctx context.Context, id uint) (bool, error) { return id == 1, nil }
+	ds.ListSoftwareCategoriesFunc = func(ctx context.Context, teamID uint) ([]fleet.SoftwareCategory, error) {
+		return slices.Clone(existing), nil
+	}
+	ds.SoftwareCategoryFunc = func(ctx context.Context, id uint) (*fleet.SoftwareCategory, error) {
+		for _, c := range existing {
+			if c.ID == id {
+				return &c, nil
+			}
+		}
+		return nil, &notFoundError{}
+	}
+	ds.DeleteSoftwareCategoryFunc = func(ctx context.Context, id uint) error {
+		deleted = append(deleted, id)
+		return nil
+	}
+
+	dir := t.TempDir()
+
+	t.Run("software excepted from gitops", func(t *testing.T) {
+		deleted = nil
+		path := filepath.Join(dir, "exception.yml")
+		require.NoError(t, os.WriteFile(path, []byte("name: Test Fleet\nteam_settings:\n  secrets:\ncontrols:\npolicies:\n"), 0o600))
+		_, err := runAppNoChecks([]string{"gitops", "-f", path})
+		require.NoError(t, err)
+		assert.Empty(t, deleted)
+	})
+
+	t.Run("non-gitops apply", func(t *testing.T) {
+		deleted = nil
+		path := filepath.Join(dir, "apply.yml")
+		require.NoError(t, os.WriteFile(path, []byte(`apiVersion: v1
+kind: fleet
+spec:
+  team:
+    name: Test Fleet
+    secrets:
+      - secret: AAA
+`), 0o600))
+		_, err := runAppNoChecks([]string{"apply", "-f", path})
+		require.NoError(t, err)
+		assert.Empty(t, deleted)
+	})
+}
+
 func TestValidateGitOpsGroupEUA(t *testing.T) {
 	t.Parallel()
 

cmd/fleetctl/fleetctl/testing_utils/testing_utils.go

@@ -502,6 +502,9 @@ func SetupFullGitOpsPremiumServer(t *testing.T) (*mock.Store, **fleet.AppConfig,
 		savedTeams[team.Name] = &team
 		return team, nil
 	}
+	ds.TeamExistsFunc = func(ctx context.Context, teamID uint) (bool, error) {
+		return true, nil
+	}
 	ds.SetOrUpdateMDMAppleDeclarationFunc = func(ctx context.Context, declaration *fleet.MDMAppleDeclaration, usesFleetVars []fleet.FleetVarName) (
 		*fleet.MDMAppleDeclaration, error,
 	) {
@@ -520,6 +523,15 @@ func SetupFullGitOpsPremiumServer(t *testing.T) (*mock.Store, **fleet.AppConfig,
 	ds.GetSoftwareInstallersPendingDeletionFunc = func(ctx context.Context, tmID *uint, incoming []fleet.SoftwareTitleIdentifier) ([]fleet.DeletedSoftwarePackage, error) {
 		return nil, nil
 	}
+	ds.ListSoftwareCategoriesFunc = func(ctx context.Context, teamID uint) ([]fleet.SoftwareCategory, error) {
+		return nil, nil
+	}
+	ds.BatchNewSoftwareCategoriesFunc = func(ctx context.Context, teamID uint, names []string) error {
+		return nil
+	}
+	ds.DeleteSoftwareCategoryFunc = func(ctx context.Context, id uint) error {
+		return nil
+	}
 
 	ds.InsertVPPTokenFunc = func(ctx context.Context, tok *fleet.VPPTokenData) (*fleet.VPPTokenDB, error) {
 		return &fleet.VPPTokenDB{}, nil

cmd/fleetctl/fleetctl/testing_utils_test.go

@@ -220,6 +220,18 @@ func setupEmptyGitOpsMocks(ds *mock.Store) {
 	ds.GetSoftwareCategoryNameToIDMapFunc = func(ctx context.Context, teamID uint, names []string) (map[string]uint, error) {
 		return nil, nil
 	}
+	ds.ListSoftwareCategoriesFunc = func(ctx context.Context, teamID uint) ([]fleet.SoftwareCategory, error) {
+		return nil, nil
+	}
+	ds.BatchNewSoftwareCategoriesFunc = func(ctx context.Context, teamID uint, names []string) error {
+		return nil
+	}
+	ds.DeleteSoftwareCategoryFunc = func(ctx context.Context, id uint) error {
+		return nil
+	}
+	ds.TeamExistsFunc = func(ctx context.Context, teamID uint) (bool, error) {
+		return true, nil
+	}
 	ds.GetTeamsWithInstallerByHashFunc = func(ctx context.Context, sha256, url string) (map[uint][]*fleet.ExistingSoftwareInstaller, error) {
 		return map[uint][]*fleet.ExistingSoftwareInstaller{}, nil
 	}