From 3ca1ff47544a60f1c9fe96ee72bfaf6a9dd133f4 Mon Sep 17 00:00:00 2001 From: jacobshandling <61553566+jacobshandling@users.noreply.github.com> Date: Wed, 26 Nov 2025 17:00:03 -0800 Subject: [PATCH] Detect UpgradeCodes when adding Windows FMA software, and persist them when the user adds that software; Fix recently introduced issue with list host software (#35876) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Related issue:** Resolves #35724 - Fixes issues 1 and 2 of the referenced bug ticket - Also fixes [this issue](https://github.com/fleetdm/fleet/pull/35739/files#r2548349172) ## Testing - [x] Added/updated automated tests - [x] QA'd all new/changed functionality manually For unreleased bug fixes in a release candidate, one of: - [x] Alerted the release DRI if additional load testing is needed ## Summary by CodeRabbit * **New Features** * Added upgrade code support to Windows maintained applications for improved MSI installer tracking and management. * **Documentation** * Updated Windows onboarding instructions with more precise manifest path guidance and concrete command examples for the maintained-apps generator. * **Tests** * Added comprehensive test coverage for upgrade code association with maintained applications. ✏️ Tip: You can customize this high-level summary in your review settings. --- ee/maintained-apps/README.md | 6 +- .../ingesters/winget/ingester.go | 8 +- ee/maintained-apps/maintained_apps.go | 1 + .../outputs/box-drive/windows.json | 3 +- .../outputs/cloudflare-warp/windows.json | 3 +- ee/maintained-apps/outputs/zoom/windows.json | 3 +- ee/server/service/maintained_apps.go | 2 +- server/datastore/mysql/software.go | 19 +- server/datastore/mysql/software_installers.go | 11 + server/fleet/maintained_apps.go | 4 +- server/mdm/maintainedapps/sync.go | 1 + server/service/integration_enterprise_test.go | 192 ++++++++++++++++++ 12 files changed, 234 insertions(+), 19 deletions(-) diff --git a/ee/maintained-apps/README.md b/ee/maintained-apps/README.md index 0df1a219d4..4d3961e463 100644 --- a/ee/maintained-apps/README.md +++ b/ee/maintained-apps/README.md @@ -49,7 +49,7 @@ ## Adding a new app (Windows) -1. Find the Winget PackageIdentifier in the [winget-pkgs repo](https://github.com/microsoft/winget-pkgs). +1. Find the Winget `PackageIdentifier` in the relevant [winget-pkgs repo manifest](https://github.com/microsoft/winget-pkgs/tree/master/manifests). 2. Get the unique identifier that Fleet will use for matching the software with software inventory: - On a test Windows host, install the app manually, then run the following PowerShell script that correlates to the defined `installer_scope`: @@ -73,7 +73,9 @@ If the `unique_identifier` doesn't match the `DisplayName`, then Fleet will inco } ``` -4. Run the following command from the root of the Fleet repo to generate the app's output data: + +4. Run `go run cmd/maintained-apps/main.go --slug="/windows" --debug` from the root of the + Fleet repo to generate the app's output data, replacing `` with your app's name, for example: ```bash go run cmd/maintained-apps/main.go --slug="box-drive/windows" --debug diff --git a/ee/maintained-apps/ingesters/winget/ingester.go b/ee/maintained-apps/ingesters/winget/ingester.go index 1a2ded5efb..e00fc538e9 100644 --- a/ee/maintained-apps/ingesters/winget/ingester.go +++ b/ee/maintained-apps/ingesters/winget/ingester.go @@ -250,8 +250,8 @@ func (i *wingetIngester) ingestOne(ctx context.Context, input inputApp) (*mainta } } + var upgradeCode string if (input.InstallerType == installerTypeMSI || input.UninstallType == installerTypeMSI) && input.InstallerScope == machineScope { - var upgradeCode string for _, fe := range m.AppsAndFeaturesEntries { if fe.UpgradeCode != "" { upgradeCode = fe.UpgradeCode @@ -288,6 +288,10 @@ func (i *wingetIngester) ingestOne(ctx context.Context, input inputApp) (*mainta } productCode = strings.Split(productCode, ".")[0] + if upgradeCode != "" { + out.UpgradeCode = upgradeCode + } + out.Name = input.Name out.Slug = input.Slug out.InstallerURL = selectedInstaller.InstallerURL @@ -306,6 +310,8 @@ func (i *wingetIngester) ingestOne(ctx context.Context, input inputApp) (*mainta if input.UniqueIdentifier != "" { name = input.UniqueIdentifier } + + // TODO - consider UpgradeCode here? existsTemplate := "SELECT 1 FROM programs WHERE name = '%s' AND publisher = '%s';" if input.FuzzyMatchName { existsTemplate = "SELECT 1 FROM programs WHERE name LIKE '%s %%' AND publisher = '%s';" diff --git a/ee/maintained-apps/maintained_apps.go b/ee/maintained-apps/maintained_apps.go index bc52dd4ef4..faa346bbe5 100644 --- a/ee/maintained-apps/maintained_apps.go +++ b/ee/maintained-apps/maintained_apps.go @@ -35,6 +35,7 @@ type FMAManifestApp struct { Name string `json:"-"` DefaultCategories []string `json:"default_categories"` Frozen bool `json:"-"` + UpgradeCode string `json:"upgrade_code,omitempty"` } func (a *FMAManifestApp) Platform() string { diff --git a/ee/maintained-apps/outputs/box-drive/windows.json b/ee/maintained-apps/outputs/box-drive/windows.json index 918633db7f..9baf7131fd 100644 --- a/ee/maintained-apps/outputs/box-drive/windows.json +++ b/ee/maintained-apps/outputs/box-drive/windows.json @@ -11,7 +11,8 @@ "sha256": "96c9631470d3f7eccd3de8ee4ee3b0c8d19adfe3e29ed4e2f9e040f1c7ad8faf", "default_categories": [ "Productivity" - ] + ], + "upgrade_code": "{46AF5B38-D258-487A-92BD-792911248CCD}" } ], "refs": { diff --git a/ee/maintained-apps/outputs/cloudflare-warp/windows.json b/ee/maintained-apps/outputs/cloudflare-warp/windows.json index 7630e81d22..09d7c02bde 100644 --- a/ee/maintained-apps/outputs/cloudflare-warp/windows.json +++ b/ee/maintained-apps/outputs/cloudflare-warp/windows.json @@ -11,7 +11,8 @@ "sha256": "ac519bcc2937d93d67d31497d72a6262801d30cea98894a9584c0b92d0635ba2", "default_categories": [ "Productivity" - ] + ], + "upgrade_code": "{1BF42825-7B65-4CA9-AFFF-B7B5E1CE27B4}" } ], "refs": { diff --git a/ee/maintained-apps/outputs/zoom/windows.json b/ee/maintained-apps/outputs/zoom/windows.json index 8dab47c228..735ed11b89 100644 --- a/ee/maintained-apps/outputs/zoom/windows.json +++ b/ee/maintained-apps/outputs/zoom/windows.json @@ -11,7 +11,8 @@ "sha256": "53e1eca704589e41a19c061bfe1bb7cd917ef230c21eb9fdd629d963f58926db", "default_categories": [ "Communication" - ] + ], + "upgrade_code": "{C819B794-A45C-4F27-9860-0C86492A52CC}" } ], "refs": { diff --git a/ee/server/service/maintained_apps.go b/ee/server/service/maintained_apps.go index 13274af0a8..5956e68d47 100644 --- a/ee/server/service/maintained_apps.go +++ b/ee/server/service/maintained_apps.go @@ -74,7 +74,6 @@ func (svc *Service) AddFleetMaintainedApp( if v := os.Getenv("FLEET_DEV_MAINTAINED_APPS_INSTALLER_TIMEOUT"); v != "" { timeout, _ = time.ParseDuration(v) } - client := fleethttp.NewClient(fleethttp.WithTimeout(timeout)) installerTFR, filename, err := maintained_apps.DownloadInstaller(ctx, app.InstallerURL, client) if err != nil { @@ -149,6 +148,7 @@ func (svc *Service) AddFleetMaintainedApp( Source: app.Source(), Extension: extension, BundleIdentifier: app.BundleIdentifier(), + UpgradeCode: app.UpgradeCode, StorageID: app.SHA256, FleetMaintainedAppID: maintainedAppID, PreInstallQuery: preInstallQuery, diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index 6ad5ea1267..76cc598a4b 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -896,7 +896,8 @@ func (ds *Datastore) preInsertSoftwareInventory( newTitlesNeeded := make(map[string]fleet.SoftwareTitle) for checksum, sw := range batchSoftware { if _, ok := incomingChecksumsToExistingTitleSummaries[checksum]; !ok { - titleName := sw.Name + // there is not an existing software title corresponding to this incoming software version + newTitleName := sw.Name if sw.BundleIdentifier != "" { key := titleKey{ bundleID: sw.BundleIdentifier, @@ -904,27 +905,27 @@ func (ds *Datastore) preInsertSoftwareInventory( extensionFor: sw.ExtensionFor, } if computedName, exists := bestTitleNames[key]; exists { - titleName = computedName + newTitleName = computedName } } - st := fleet.SoftwareTitle{ - Name: titleName, + newTitle := fleet.SoftwareTitle{ + Name: newTitleName, Source: sw.Source, ExtensionFor: sw.ExtensionFor, IsKernel: sw.IsKernel, } if sw.BundleIdentifier != "" { - st.BundleIdentifier = ptr.String(sw.BundleIdentifier) + newTitle.BundleIdentifier = ptr.String(sw.BundleIdentifier) } if sw.ApplicationID != nil && *sw.ApplicationID != "" { - st.ApplicationID = sw.ApplicationID + newTitle.ApplicationID = sw.ApplicationID } if sw.UpgradeCode != nil { // intentionally write both empty and non-empty strings as upgrade codes - st.UpgradeCode = sw.UpgradeCode + newTitle.UpgradeCode = sw.UpgradeCode } - newTitlesNeeded[checksum] = st + newTitlesNeeded[checksum] = newTitle } } @@ -2915,7 +2916,6 @@ func (ds *Datastore) ListCVEs(ctx context.Context, maxAge time.Duration) ([]flee return result, nil } -// TODO(jacob) SoftwareUpgradeCode ? SoftwareUpgradeCodeList ? type hostSoftware struct { fleet.HostSoftwareWithInstaller @@ -2931,7 +2931,6 @@ type hostSoftware struct { SoftwareID *uint `db:"software_id"` SoftwareSource *string `db:"software_source"` SoftwareExtensionFor *string `db:"software_extension_for"` - UpgradeCode *string `db:"upgrade_code"` InstallerID *uint `db:"installer_id"` PackageSelfService *bool `db:"package_self_service"` PackageName *string `db:"package_name"` diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index a543f1ea40..f16b4df20b 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -507,10 +507,18 @@ func (ds *Datastore) getOrGenerateSoftwareInstallerTitleID(ctx context.Context, insertStmt := `INSERT INTO software_titles (name, source, extension_for) VALUES (?, ?, '')` insertArgs := []any{payload.Title, payload.Source} + // upgrade_code should be NULL for non-Windows software, empty or non-empty string for Windows + // software + if payload.Source == "programs" { + insertStmt = `INSERT INTO software_titles (name, source, extension_for, upgrade_code) VALUES (?, ?, '', ?)` + insertArgs = []any{payload.Title, payload.Source, payload.UpgradeCode} + } + if payload.BundleIdentifier != "" { // match by bundle identifier first, or standard matching if we don't have a bundle identifier match selectStmt = `SELECT id FROM software_titles WHERE bundle_identifier = ? OR (name = ? AND source = ? AND extension_for = '') ORDER BY bundle_identifier = ? DESC LIMIT 1` selectArgs = []any{payload.BundleIdentifier, payload.Title, payload.Source, payload.BundleIdentifier} + // omit upgrade_code, since title.upgrade_code should be NULL for non-Windows software insertStmt = `INSERT INTO software_titles (name, source, bundle_identifier, extension_for) VALUES (?, ?, ?, '')` insertArgs = append(insertArgs, payload.BundleIdentifier) } @@ -532,6 +540,9 @@ func (ds *Datastore) getOrGenerateSoftwareInstallerTitleID(ctx context.Context, } func (ds *Datastore) addSoftwareTitleToMatchingSoftware(ctx context.Context, titleID uint, payload *fleet.UploadSoftwareInstallerPayload) error { + // not considering upgrade_code, so inventory software will match this Title by this clause, even + // if upgrade_code doesn't match - TODO: enforce matching upgrade_code between software and + // incoming title? whereClause := "WHERE (s.name, s.source, s.extension_for) = (?, ?, '')" whereArgs := []any{payload.Title, payload.Source} if payload.BundleIdentifier != "" { diff --git a/server/fleet/maintained_apps.go b/server/fleet/maintained_apps.go index 4ae9ca26dc..a968142fb8 100644 --- a/server/fleet/maintained_apps.go +++ b/server/fleet/maintained_apps.go @@ -2,8 +2,7 @@ package fleet import "net/http" -// MaintainedApp represents an app in the Fleet library of maintained apps, -// as stored in the fleet_library_apps table. +// MaintainedApp represents an app in the Fleet library of maintained apps type MaintainedApp struct { ID uint `json:"id" db:"id"` Name string `json:"name" db:"name"` @@ -18,6 +17,7 @@ type MaintainedApp struct { UninstallScript string `json:"uninstall_script,omitempty"` AutomaticInstallQuery string `json:"-"` Categories []string `json:"categories"` + UpgradeCode string `json:"upgrade_code,omitempty"` } func (s *MaintainedApp) Source() string { diff --git a/server/mdm/maintainedapps/sync.go b/server/mdm/maintainedapps/sync.go index 3ee2513e86..41ed00dfc0 100644 --- a/server/mdm/maintainedapps/sync.go +++ b/server/mdm/maintainedapps/sync.go @@ -153,6 +153,7 @@ func Hydrate(ctx context.Context, app *fleet.MaintainedApp) (*fleet.MaintainedAp app.UninstallScript = manifest.Refs[manifest.Versions[0].UninstallScriptRef] app.AutomaticInstallQuery = manifest.Versions[0].Queries.Exists app.Categories = manifest.Versions[0].DefaultCategories + app.UpgradeCode = manifest.Versions[0].UpgradeCode return app, nil } diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 5def3116f7..34ae1b4661 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -53,6 +53,7 @@ import ( "github.com/fleetdm/fleet/v4/server/pubsub" commonCalendar "github.com/fleetdm/fleet/v4/server/service/calendar" "github.com/fleetdm/fleet/v4/server/service/conditional_access_microsoft_proxy" + "github.com/fleetdm/fleet/v4/server/service/osquery_utils" "github.com/fleetdm/fleet/v4/server/service/redis_lock" "github.com/fleetdm/fleet/v4/server/service/schedule" "github.com/fleetdm/fleet/v4/server/test" @@ -18179,6 +18180,197 @@ func (s *integrationEnterpriseTestSuite) TestMaintainedApps() { require.Contains(t, extractServerErrorText(r.Body), `Only one of "labels_include_any" or "labels_exclude_any" can be included`) } +func (s *integrationEnterpriseTestSuite) TestUpgradeCodesFromMaintainedApps() { + // Specifically test that `upgrade_code` is correctly associated with host software that has + // first been added via FMA. For a more robust handling of possible error scenarios when adding + // Maintained Apps, see `TestMaintainedApps` + + t := s.T() + ctx := context.Background() + + warpUpgradeCode := "{1BF42825-7B65-4CA9-AFFF-B7B5E1CE27B4}" + + ac, err := s.ds.AppConfig(ctx) + require.NoError(s.T(), err) + ac.Features.EnableSoftwareInventory = true + err = s.ds.SaveAppConfig(context.Background(), ac) + require.NoError(s.T(), err) + time.Sleep(2 * time.Second) // Wait for the app config cache to clear + + installerBytes := []byte("abc") + installerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/badinstaller": + _, _ = w.Write([]byte("badinstaller")) + case "/timeout": + time.Sleep(3 * time.Second) + _, _ = w.Write([]byte("timeout")) + default: + _, _ = w.Write(installerBytes) + } + })) + defer installerServer.Close() + + // Mock server to serve manifest with no_check + manifestServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var versions []*ma.FMAManifestApp + versions = append(versions, &ma.FMAManifestApp{ + Version: "25.9.558.0", + Queries: ma.FMAQueries{ + Exists: "SELECT 1 FROM osquery_info;", + }, + InstallerURL: installerServer.URL + "/installer.msi", + InstallScriptRef: "foobaz", + UninstallScriptRef: "foobaz", + SHA256: "no_check", + UpgradeCode: warpUpgradeCode, + }) + + manifest := ma.FMAManifestFile{ + Versions: versions, + Refs: map[string]string{ + "foobaz": "Hello World!", + }, + } + + err := json.NewEncoder(w).Encode(manifest) + require.NoError(t, err) + })) + t.Cleanup(manifestServer.Close) + + mockTransport := &mockRoundTripper{ + mockServer: manifestServer.URL, + origBaseURL: "https://raw.githubusercontent.com", + next: http.DefaultTransport, + } + http.DefaultTransport = mockTransport + + // Insert the list of maintained apps + maintained_apps.SyncApps(t, s.ds) + + // verify WARP is in `fleet_maintained_apps` table but not in `software_installers` + var warpFmaId uint + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(ctx, q, &warpFmaId, "SELECT id FROM fleet_maintained_apps WHERE name = 'Cloudflare WARP' AND platform = 'windows'") + }) + require.NotNil(t, warpFmaId) + + var count uint + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(ctx, q, &count, "SELECT COUNT(*) FROM software_installers") + }) + require.Zero(t, count) + + // Create a team + var newTeamResp teamResponse + s.DoJSON("POST", "/api/latest/fleet/teams", &createTeamRequest{TeamPayload: fleet.TeamPayload{Name: ptr.String("Team 1")}}, http.StatusOK, &newTeamResp) + team := newTeamResp.Team + + // Add WARP for Windows + var warpAppId uint + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(ctx, q, &warpAppId, "SELECT id FROM fleet_maintained_apps WHERE name = 'Cloudflare WARP' and platform = 'windows'") + }) + + var addMAresp addFleetMaintainedAppResponse + req := &addFleetMaintainedAppRequest{ + AppID: warpAppId, + TeamID: &team.ID, + SelfService: true, + PreInstallQuery: "SELECT 1", + PostInstallScript: "echo done", + } + + s.DoJSON("POST", "/api/latest/fleet/software/fleet_maintained_apps", req, http.StatusOK, &addMAresp) + require.Nil(t, addMAresp.Err) + + // Verify WARP is now in `software_installers`, a `software_tiles` row has been created, and they + // are associated + var warpInstaller fleet.SoftwareInstaller + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(ctx, q, &warpInstaller, "SELECT id, title_id, upgrade_code, filename FROM software_installers WHERE upgrade_code = ?", warpUpgradeCode) + }) + require.NotNil(t, warpInstaller) + require.NotNil(t, warpInstaller.Name) + + var lSTResp listSoftwareTitlesResponse + s.DoJSON( + "GET", "/api/latest/fleet/software/titles", + listSoftwareTitlesRequest{}, + http.StatusOK, &lSTResp, + "per_page", "1", + "order_key", "name", + "order_direction", "desc", + "available_for_install", "true", + "team_id", fmt.Sprintf("%d", team.ID), + ) + title := lSTResp.SoftwareTitles[0] + require.Equal(t, *warpInstaller.TitleID, title.ID) + require.Equal(t, warpUpgradeCode, *title.UpgradeCode) + + // Create a Windows host on the team + host, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now().Add(-1 * time.Minute), + OsqueryHostID: ptr.String(t.Name()), + NodeKey: ptr.String(t.Name()), + UUID: uuid.New().String(), + Hostname: fmt.Sprintf("%sfoo.local", t.Name()), + Platform: "windows", + TeamID: &team.ID, + }) + require.NoError(t, err) + + // mock osquery ingesting matching software from the host + ac, err = s.ds.AppConfig(context.Background()) + require.NoError(t, err) + + detailQueries := osquery_utils.GetDetailQueries(context.Background(), config.FleetConfig{}, ac, &ac.Features, osquery_utils.Integrations{}, nil) + + rows := []map[string]string{ + { + "name": "Cloudflare WARP", + "version": "25.9.558.0", + "source": "programs", + "vendor": "Cloudflare, Inc.", + "upgrade_code": warpUpgradeCode, + }, + } + + err = detailQueries["software_windows"].DirectIngestFunc( + context.Background(), + kitlog.NewNopLogger(), + &fleet.Host{ID: host.ID}, + s.ds, + rows, + ) + require.NoError(t, err) + + // confirm software row now in the database and `upgrade_code`s match for that row, the associated + // `software_titles` row, and the associated `software_installers` row + var swTitleId *uint + mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(ctx, q, &swTitleId, "SELECT title_id FROM software WHERE upgrade_code = ?", warpUpgradeCode) + }) + require.Equal(t, title.ID, *swTitleId) + require.Equal(t, *warpInstaller.TitleID, *swTitleId) + + // GET host software endpoint, confirm upgrade_code is present + var hSWRes getHostSoftwareResponse + s.DoJSON( + "GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", host.ID), + getHostSoftwareRequest{}, + http.StatusOK, &hSWRes, + ) + // should only be the single software and software title in the database + require.Equal(t, hSWRes.Count, 1) + require.Equal(t, len(hSWRes.Software), 1) + sw0 := hSWRes.Software[0] + require.Equal(t, warpUpgradeCode, *sw0.UpgradeCode) +} + func (s *integrationEnterpriseTestSuite) TestWindowsMigrateMDMNotEnabled() { t := s.T()