Add "generate-gitops" command (#28555)

For #27476

# Checklist for submitter

- [X] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/Committing-Changes.md#changes-files)
for more information.
- [X] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements)

# Details

This PR adds a new command `generate-gitops` to the `fleetctl` tool. The
purpose of this command is to output GitOps-ready files that can then be
used with `fleetctl-gitops`.

The general usage of the command is:

```
fleectl generate-gitops --dir /path/to/dir/to/add/files/to
```

By default, the outputted files will not contain sensitive data, but
will instead add comments where the data needs to be replaced by a user.
In cases where sensitive data is redacted, the tool outputs warnings to
the user indicating which keys need to be updated.

The tool uses existing APIs to gather data for use in generating
configuration files. In some cases new API client methods needed to be
added to support the tool:

* ListConfigurationProfiles
* GetProfileContents
* GetScriptContents
* GetSoftwareTitleByID

Additionally, the response for the /api/latest/fleet/software/batch
endpoint was updated slightly to return `HashSHA256` for the software
installers. This allows policies that automatically install software to
refer to that software by hash.

Other options that we may or may not choose to document at this time:

* `--insecure`: outputs sensitive data in plaintext instead of leaving
comments
* `--print`: prints the output to stdout instead of writing files
* `--key`: outputs the value at a keypath to stdout, e.g. `--key
agent_options.config`
* `--team`: only generates config for the specified team name
* `--force`: overwrites files in the given directory (defaults to false,
which errors if the dir is not empty)

# Technical notes

The command is implemented using a `GenerateGitopsCommand` type which
holds some state (like a list of software and scripts encountered) as
well as a Fleet client instance (which may be a mock instance for tests)
and the CLI context (containing things like flags and output writers).
The actual "action" of the CLI command calls the `Run()` method of the
`GenerateGitopsCommand` var, which delegates most of the work to other
methods like `generateOrgSettings()`, `generateControls()`, etc.

Wherever possible, the subroutines use reflection to translate Go struct
fields into JSON property names. This guarantees that the correct keys
are written to config files, and protects against the unlikely event of
keys changing.

When sensitive data is encountered, the subroutines call `AddComment()`
to get a new token to add to the config files. These tokens are replaced
with comments like `# TODO - Add your enrollment secrets here` in the
final output.

# Known issues / TODOs:

* The `macos_setup` configuration is not output by this tool yet. More
planning is required for this. In the meantime, if the tool detects that
`macos_setup` is configured on the server, it outputs a key with an
invalid value and prints a warning to the user that they'll need to
configure it themselves.
* `yara_rules` are not output yet. The tool adds a warning that if you
have Yara rules (which you can only upload via GitOps right now) that
you'll have to migrate them manually. Supporting this will require a new
API that we'll have to discuss the authz for, so punting on it for now.
* Fleet maintained apps are not supported by GitOps yet (coming in
https://github.com/fleetdm/fleet/issues/24469). In the meantime, this
tool will output a `fleet_maintained_apps` key and trigger a warning,
and GitOps will fail if that key is present.

---------

Co-authored-by: Lucas Manuel Rodriguez <lucas@fleetdm.com>
Co-authored-by: Noah Talerman <47070608+noahtalerman@users.noreply.github.com>

server/datastore/mysql/software_installers.go

@@ -654,6 +654,7 @@ SELECT
   si.team_id,
   si.title_id,
   si.storage_id,
+  si.fleet_maintained_app_id,
   si.package_ids,
   si.filename,
   si.extension,
@@ -2166,7 +2167,9 @@ func (ds *Datastore) GetSoftwareInstallers(ctx context.Context, teamID uint) ([]
 SELECT
   team_id,
   title_id,
-  url
+  url,
+  storage_id as hash_sha256,
+  fleet_maintained_app_id
 FROM
   software_installers
 WHERE global_or_team_id = ?

server/datastore/mysql/software_titles.go

@@ -321,6 +321,7 @@ SELECT
 	si.url AS package_url,
 	si.install_during_setup as package_install_during_setup,
 	si.storage_id as package_storage_id,
+	si.fleet_maintained_app_id,
 	vat.self_service as vpp_app_self_service,
 	vat.adam_id as vpp_app_adam_id,
 	vat.install_during_setup as vpp_install_during_setup,
@@ -338,7 +339,7 @@ LEFT JOIN software_titles_host_counts sthc ON sthc.software_title_id = st.id AND
 WHERE %s
 -- placeholder for filter based on software installed on hosts + software installers
 AND (%s)
-GROUP BY st.id, package_self_service, package_name, package_version, package_platform, package_url, package_install_during_setup, package_storage_id, vpp_app_self_service, vpp_app_adam_id, vpp_app_version, vpp_app_platform, vpp_app_icon_url, vpp_install_during_setup`
+GROUP BY st.id, package_self_service, package_name, package_version, package_platform, package_url, package_install_during_setup, package_storage_id, fleet_maintained_app_id, vpp_app_self_service, vpp_app_adam_id, vpp_app_version, vpp_app_platform, vpp_app_icon_url, vpp_install_during_setup`
 
 	cveJoinType := "LEFT"
 	if opt.VulnerableOnly {

server/fleet/software.go

@@ -239,8 +239,9 @@ type SoftwareTitleListResult struct {
 	// BundleIdentifier is used by Apple installers to uniquely identify
 	// the software installed. It's surfaced in software_titles to match
 	// with existing software entries.
-	BundleIdentifier *string `json:"bundle_identifier,omitempty" db:"bundle_identifier"`
-	HashSHA256       *string `json:"hash_sha256,omitempty" db:"package_storage_id"`
+	BundleIdentifier     *string `json:"bundle_identifier,omitempty" db:"bundle_identifier"`
+	HashSHA256           *string `json:"hash_sha256,omitempty" db:"package_storage_id"`
+	FleetMaintainedAppID *uint   `json:"fleet_maintained_app_id,omitempty" db:"fleet_maintained_app_id"`
 }
 
 type SoftwareTitleListOptions struct {

server/fleet/software_installer.go

@@ -137,7 +137,7 @@ type SoftwareInstaller struct {
 	// URL is the source URL for this installer (set when uploading via batch/gitops).
 	URL string `json:"url" db:"url"`
 	// FleetMaintainedAppID is the related Fleet-maintained app for this installer (if not nil).
-	FleetMaintainedAppID *uint `json:"-" db:"fleet_maintained_app_id"`
+	FleetMaintainedAppID *uint `json:"fleet_maintained_app_id" db:"fleet_maintained_app_id"`
 	// AutomaticInstallPolicies is the list of policies that trigger automatic
 	// installation of this software.
 	AutomaticInstallPolicies []AutomaticInstallPolicy `json:"automatic_install_policies" db:"-"`
@@ -161,6 +161,10 @@ type SoftwarePackageResponse struct {
 	TitleID *uint `json:"title_id" db:"title_id"`
 	// URL is the source URL for this installer (set when uploading via batch/gitops).
 	URL string `json:"url" db:"url"`
+	// HashSHA256 is the SHA256 hash of the software installer.
+	HashSHA256 string `json:"hash_sha256" db:"hash_sha256"`
+	// ID of the Fleet Maintained App this package uses, if any
+	FleetMaintainedAppID *uint `json:"fleet_maintained_app_id" db:"fleet_maintained_app_id"`
 }
 
 // VPPAppResponse is the response type used when applying app store apps by batch.

server/service/client.go

@@ -2097,18 +2097,20 @@ func (c *Client) doGitOpsPolicies(config *spec.GitOps, teamSoftwareInstallers []
 		// Get software titles of packages for the team.
 		softwareTitleIDsByInstallerURL := make(map[string]uint)
 		softwareTitleIDsByAppStoreAppID := make(map[string]uint)
+		softwareTitleIDsByHash := make(map[string]uint)
 		for _, softwareInstaller := range teamSoftwareInstallers {
 			if softwareInstaller.TitleID == nil {
 				// Should not happen, but to not panic we just log a warning.
 				logFn("[!] software installer without title id: team_id=%d, url=%s\n", *teamID, softwareInstaller.URL)
 				continue
 			}
-			if softwareInstaller.URL == "" {
+			if softwareInstaller.URL == "" && softwareInstaller.HashSHA256 == "" {
 				// Should not happen because we previously applied packages via gitops, but to not panic we just log a warning.
 				logFn("[!] software installer without url: team_id=%d, title_id=%d\n", *teamID, *softwareInstaller.TitleID)
 				continue
 			}
 			softwareTitleIDsByInstallerURL[softwareInstaller.URL] = *softwareInstaller.TitleID
+			softwareTitleIDsByHash[softwareInstaller.HashSHA256] = *softwareInstaller.TitleID
 		}
 		for _, vppApp := range teamVPPApps {
 			if vppApp.Platform != fleet.MacOSPlatform {
@@ -2155,6 +2157,17 @@ func (c *Client) doGitOpsPolicies(config *spec.GitOps, teamSoftwareInstallers []
 				}
 				config.Policies[i].SoftwareTitleID = &softwareTitleID
 			}
+			if config.Policies[i].InstallSoftware.HashSHA256 != "" {
+				softwareTitleID, ok := softwareTitleIDsByHash[config.Policies[i].InstallSoftware.HashSHA256]
+				if !ok {
+					// Should not happen because software packages are uploaded first.
+					if !dryRun {
+						logFn("[!] software hash without software title ID: %s\n", config.Policies[i].InstallSoftware.HashSHA256)
+					}
+					continue
+				}
+				config.Policies[i].SoftwareTitleID = &softwareTitleID
+			}
 		}
 
 		// Get scripts for the team.

server/service/client_profiles.go

@@ -36,6 +36,41 @@ func (c *Client) ListProfiles(teamID *uint) ([]*fleet.MDMAppleConfigProfile, err
 	return responseBody.ConfigProfiles, nil
 }
 
+func (c *Client) ListConfigurationProfiles(teamID *uint) ([]*fleet.MDMConfigProfilePayload, error) {
+	verb, path := "GET", "/api/latest/fleet/configuration_profiles"
+	query := make(url.Values)
+	if teamID != nil {
+		query.Add("team_id", strconv.FormatUint(uint64(*teamID), 10))
+	}
+	var responseBody listMDMConfigProfilesResponse
+	if err := c.authenticatedRequestWithQuery(nil, verb, path, &responseBody, query.Encode()); err != nil {
+		return nil, err
+	}
+	return responseBody.Profiles, nil
+}
+
+// Get the contents of a saved profile.
+func (c *Client) GetProfileContents(profileID string) ([]byte, error) {
+	verb, path := "GET", "/api/latest/fleet/mdm/profiles/"+profileID
+	response, err := c.AuthenticatedDo(verb, path, "alt=media", nil)
+	if err != nil {
+		return nil, fmt.Errorf("%s %s: %w", verb, path, err)
+	}
+	defer response.Body.Close()
+	err = c.parseResponse(verb, path, response, nil)
+	if err != nil {
+		return nil, fmt.Errorf("%s %s: %w", verb, path, err)
+	}
+	if response.StatusCode != http.StatusNoContent {
+		b, err := io.ReadAll(response.Body)
+		if err != nil {
+			return nil, fmt.Errorf("reading response body: %w", err)
+		}
+		return b, nil
+	}
+	return nil, nil
+}
+
 func (c *Client) AddProfile(teamID uint, configurationProfile []byte) (uint, error) {
 	if c.token == "" {
 		return 0, errors.New("authentication token is empty")

server/service/client_scripts.go

@@ -229,3 +229,25 @@ func (c *Client) ListScripts(query string) ([]*fleet.Script, error) {
 	}
 	return responseBody.Scripts, nil
 }
+
+// Get the contents of a saved script.
+func (c *Client) GetScriptContents(scriptID uint) ([]byte, error) {
+	verb, path := "GET", "/api/latest/fleet/scripts/"+fmt.Sprint(scriptID)
+	response, err := c.AuthenticatedDo(verb, path, "alt=media", nil)
+	if err != nil {
+		return nil, fmt.Errorf("%s %s: %w", verb, path, err)
+	}
+	defer response.Body.Close()
+	err = c.parseResponse(verb, path, response, nil)
+	if err != nil {
+		return nil, fmt.Errorf("parsing script response: %w", err)
+	}
+	if response.StatusCode != http.StatusNoContent {
+		b, err := io.ReadAll(response.Body)
+		if err != nil {
+			return nil, fmt.Errorf("reading response body: %w", err)
+		}
+		return b, nil
+	}
+	return nil, nil
+}

server/service/client_software.go

@@ -31,6 +31,23 @@ func (c *Client) ListSoftwareTitles(query string) ([]fleet.SoftwareTitleListResu
 	return responseBody.SoftwareTitles, nil
 }
 
+// GetSoftwareTitleByID retrieves a software title by ID.
+//
+//nolint:gocritic // ignore captLocal
+func (c *Client) GetSoftwareTitleByID(ID uint, teamID *uint) (*fleet.SoftwareTitle, error) {
+	var query string
+	if teamID != nil {
+		query = fmt.Sprintf("team_id=%d", *teamID)
+	}
+	verb, path := "GET", "/api/latest/fleet/software/titles/"+fmt.Sprint(ID)
+	var responseBody getSoftwareTitleResponse
+	err := c.authenticatedRequestWithQuery(nil, verb, path, &responseBody, query)
+	if err != nil {
+		return nil, err
+	}
+	return responseBody.SoftwareTitle, nil
+}
+
 func (c *Client) ApplyNoTeamSoftwareInstallers(softwareInstallers []fleet.SoftwareInstallerPayload, opts fleet.ApplySpecOptions) ([]fleet.SoftwarePackageResponse, error) {
 	query, err := url.ParseQuery(opts.RawQuery())
 	if err != nil {