Ingest, store, consider in unique_identifier, and serve upgrade_codes for Windows software (#34786)

<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #33907 

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`
- [x] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements)

## Testing

- [x] Added/updated automated tests
- [x] QA'd all new/changed functionality manually

## Database migrations

- [x] Checked schema for all modified table for columns that will
auto-update timestamps during migration.
~- [ ] Confirmed that updating the timestamps is acceptable, and will
not cause unwanted side effects.~ N/A
- [x] Ensured the correct collation is explicitly set for character
columns (`COLLATE utf8mb4_unicode_ci`).


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

## Release Notes

* **New Features**
* Windows software inventory now includes upgrade code data for better
software identification and tracking.

* **Chores**
* Database schema updated to support upgrade code storage for software
titles and inventory records.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

server/datastore/mysql/migrations/tables/20251107170854_AddWindowsUpgradeCodeToSoftware.go

@@ -0,0 +1,45 @@
+package tables
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20251107170854, Down_20251107170854)
+}
+
+func Up_20251107170854(tx *sql.Tx) error {
+	// CHAR(38) to account for 32 hex chars + 4 hyphens + open/close curly braces
+	_, err := tx.Exec(`ALTER TABLE software_titles ADD COLUMN upgrade_code CHAR(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL`)
+	if err != nil {
+		return fmt.Errorf("failed to add software_titles.upgrade_code column: %w", err)
+	}
+	_, err = tx.Exec(`UPDATE software_titles SET upgrade_code = '' WHERE source = 'programs'`)
+	if err != nil {
+		return fmt.Errorf("failed to add default empty string value to software_titles.upgrade_code column for rows where source = 'programs': %w", err)
+	}
+
+	_, err = tx.Exec(`ALTER TABLE software ADD COLUMN upgrade_code CHAR(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL`)
+	if err != nil {
+		return fmt.Errorf("failed to add software.upgrade_code column: %w", err)
+	}
+
+	_, err = tx.Exec(`UPDATE software SET upgrade_code = '' WHERE source = 'programs'`)
+	if err != nil {
+		return fmt.Errorf("failed to add default empty string value to software.upgrade_code column for rows where source = 'programs': %w", err)
+	}
+
+	// NULLIF(upgrade_code, "") prevents upgrade_code being used as the unique_identifier when it is
+	// the empty string, which will be the case for "programs"-sourced software but is obviously not unique
+	_, err = tx.Exec(`ALTER TABLE software_titles MODIFY COLUMN unique_identifier VARCHAR(255) GENERATED ALWAYS AS (COALESCE(bundle_identifier, application_id, NULLIF(upgrade_code, ""), name)) VIRTUAL`)
+	if err != nil {
+		return fmt.Errorf("failed to alter definition of software_titles.unique_identifier column to include upgrade_code in its COALESCE: %w", err)
+	}
+
+	return nil
+}
+
+func Down_20251107170854(tx *sql.Tx) error {
+	return nil
+}

server/datastore/mysql/migrations/tables/20251107170854_AddWindowsUpgradeCodeToSoftware_test.go

@@ -0,0 +1,127 @@
+package tables
+
+import (
+	"testing"
+
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/fleetdm/fleet/v4/server/ptr"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUp_20251107170854(t *testing.T) {
+	db := applyUpToPrev(t)
+
+	ms := fleet.SoftwareTitle{
+		Name:             "iTerm.app",
+		Source:           "apps",
+		BundleIdentifier: ptr.String("com.googlecode.iterm2"),
+		UpgradeCode:      nil,
+	}
+
+	ws1 := fleet.SoftwareTitle{
+		Name:        "Notepad",
+		Source:      "programs",
+		UpgradeCode: ptr.String("{1BF42825-7B65-4CA9-AFFF-B7B5E1CE27B4}"),
+	}
+
+	ws2 := fleet.SoftwareTitle{
+		Name:        "NoteFad",
+		Source:      "programs",
+		UpgradeCode: ptr.String(""),
+	}
+
+	// Add Mac and Windows software, no upgrade codes yet. The unique_identifier should be the bundle_identifier for the
+	// macOS software and the name for the Windows software.
+
+	// these type conversions are safe from integer overflow since they are all sourced from database
+	// auto-incremented ids, which there will only be a small amount of in the context of this test
+	ms.ID = uint(execNoErrLastID(t, db, `INSERT INTO software_titles (name, source, bundle_identifier) VALUES (?, ?, ?)`, ms.Name, ms.Source, ms.BundleIdentifier)) //nolint:gosec // dismiss G115
+	ws1.ID = uint(execNoErrLastID(t, db, `INSERT INTO software_titles (name, source) VALUES (?, ?)`, ws1.Name, ws1.Source))                                         //nolint:gosec // dismiss G115
+	ws2.ID = uint(execNoErrLastID(t, db, `INSERT INTO software_titles (name, source) VALUES (?, ?)`, ws2.Name, ws2.Source))                                         //nolint:gosec // dismiss G115
+
+	// // //
+	// Apply current migration.
+	applyNext(t, db)
+	// // //
+
+	// Check default values are set as expected
+	var winUC *string
+	err := db.Get(&winUC, `SELECT upgrade_code FROM software_titles WHERE id = ?`, ws1.ID)
+	require.NoError(t, err)
+	require.Equal(t, "", *winUC)
+
+	err = db.Get(&winUC, `SELECT upgrade_code FROM software_titles WHERE id = ?`, ws2.ID)
+	require.NoError(t, err)
+	require.Equal(t, "", *winUC)
+
+	var macUC *string
+	err = db.Get(&macUC, `SELECT upgrade_code FROM software_titles WHERE id = ?`, ms.ID)
+	require.NoError(t, err)
+	require.Nil(t, macUC)
+
+	// Delete the existing Windows software, then them back now with one empty and one non-empty upgrade_code
+	execNoErr(t, db, `DELETE FROM software_titles WHERE id IN (?, ?)`, ws1.ID, ws2.ID)
+
+	ws1.ID = uint(execNoErrLastID(t, db, `INSERT INTO software_titles (name, source, upgrade_code) VALUES (?, ?, ?)`, ws1.Name, ws1.Source, ws1.UpgradeCode)) //nolint:gosec // dismiss G115
+	ws2.ID = uint(execNoErrLastID(t, db, `INSERT INTO software_titles (name, source, upgrade_code) VALUES (?, ?, ?)`, ws2.Name, ws2.Source, ws2.UpgradeCode)) //nolint:gosec // dismiss G115
+
+	cases := []struct {
+		name                string
+		titleID             uint
+		source              string
+		expectedBundleID    *string
+		expectedUpgradeCode *string
+		expectedUniqueID    string
+	}{
+		{
+			name:                "macSW",
+			titleID:             ms.ID,
+			source:              ms.Source,
+			expectedBundleID:    ms.BundleIdentifier,
+			expectedUpgradeCode: ms.UpgradeCode,       // nil
+			expectedUniqueID:    *ms.BundleIdentifier, // expect COALESCE to choose populated bundle id
+		},
+		{
+			name:                "winSW with UC",
+			titleID:             ws1.ID,
+			source:              ws1.Source,
+			expectedBundleID:    nil,
+			expectedUpgradeCode: ws1.UpgradeCode,
+			expectedUniqueID:    *ws1.UpgradeCode, // expect COALESCE to choose populated upgrade code
+		},
+		{
+			name:                "winSW no UC",
+			titleID:             ws2.ID,
+			source:              ws2.Source,
+			expectedBundleID:    nil,
+			expectedUpgradeCode: ws2.UpgradeCode, // ""
+			expectedUniqueID:    ws2.Name,        // expect NULLIF to nullify "" so COALESCE chooses the software name
+		},
+	}
+
+	for _, tC := range cases {
+		t.Run(tC.name, func(t *testing.T) {
+			var title fleet.SoftwareTitle
+			err := db.Get(&title, `SELECT id, source, bundle_identifier, upgrade_code FROM software_titles WHERE id = ?`, tC.titleID)
+			require.NoError(t, err)
+			if title.ID == ms.ID {
+				// mac
+				require.Nil(t, title.UpgradeCode)
+				require.NotNil(t, title.BundleIdentifier)
+				assert.Equal(t, *tC.expectedBundleID, *title.BundleIdentifier)
+			} else {
+				// windows
+				require.Nil(t, title.BundleIdentifier)
+				require.NotNil(t, title.UpgradeCode)
+				assert.Equal(t, *tC.expectedUpgradeCode, *title.UpgradeCode)
+			}
+
+			var gotUniqueID string
+			err = db.Get(&gotUniqueID, "SELECT unique_identifier FROM software_titles WHERE id = ?", tC.titleID)
+			require.NoError(t, err)
+
+			assert.Equal(t, tC.expectedUniqueID, gotUniqueID)
+		})
+	}
+}

server/datastore/mysql/software.go

@@ -27,12 +27,13 @@ import (
 	"go.opentelemetry.io/otel/trace"
 )
 
-type softwareIDChecksum struct {
+type softwareSummary struct {
 	ID               uint    `db:"id"`
 	Checksum         string  `db:"checksum"`
 	Name             string  `db:"name"`
 	TitleID          *uint   `db:"title_id"`
 	BundleIdentifier *string `db:"bundle_identifier"`
+	UpgradeCode      *string `db:"upgrade_code"`
 	Source           string  `db:"source"`
 }
 
@@ -351,6 +352,7 @@ SELECT
     s.vendor,
     s.arch,
     s.extension_id,
+    s.upgrade_code,
     hs.last_opened_at
 FROM
     software s
@@ -385,16 +387,16 @@ func filterSoftwareWithEmptyNames(software []fleet.Software) []fleet.Software {
 func (ds *Datastore) applyChangesForNewSoftwareDB(
 	ctx context.Context,
 	hostID uint,
-	software []fleet.Software,
+	incomingSoftware []fleet.Software,
 ) (*fleet.UpdateHostSoftwareDBResult, error) {
 	r := &fleet.UpdateHostSoftwareDBResult{}
 
 	// We want to make sure we have valid data before proceeding. We've seen Windows programs with empty names.
-	software = filterSoftwareWithEmptyNames(software)
+	incomingSoftware = filterSoftwareWithEmptyNames(incomingSoftware)
 
-	// This code executes once an hour for each host, so we should optimize for MySQL master (writer) DB performance.
-	// We use a slave (reader) DB to avoid accessing the master. If nothing has changed, we avoid all access to the master.
-	// It is possible that the software list is out of sync between the slave and the master. This is unlikely because
+	// This code executes once an hour for each host, so we should optimize for MySQL writer DB performance.
+	// We use a reader DB to avoid accessing the writer. If nothing has changed, we avoid all access to the writer.
+	// It is possible that the software list is out of sync between the reader and the writer. This is unlikely because
 	// it is updated once an hour under normal circumstances. If this does occur, the software list will be updated
 	// once again in an hour.
 	currentSoftware, err := listSoftwareByHostIDShort(ctx, ds.reader(ctx), hostID)
@@ -403,12 +405,12 @@ func (ds *Datastore) applyChangesForNewSoftwareDB(
 	}
 	r.WasCurrInstalled = currentSoftware
 
-	current, incoming, notChanged := nothingChanged(currentSoftware, software, ds.minLastOpenedAtDiff)
-	if notChanged {
+	current, incoming, noChanges := nothingChanged(currentSoftware, incomingSoftware, ds.minLastOpenedAtDiff)
+	if noChanges {
 		return r, nil
 	}
 
-	existingSoftware, incomingByChecksum, existingTitlesForNewSoftware, err := ds.getExistingSoftware(ctx, current, incoming)
+	existingSoftwareSummaries, incomingSoftwareByChecksum, incomingChecksumsToExistingTitles, err := ds.getExistingSoftware(ctx, current, incoming)
 	if err != nil {
 		return r, err
 	}
@@ -417,9 +419,9 @@ func (ds *Datastore) applyChangesForNewSoftwareDB(
 	// This reduces lock contention by breaking up large INSERT IGNORE operations
 	// into smaller, faster transactions that release locks quickly.
 	// These operations are idempotent due to INSERT IGNORE.
-	if len(incomingByChecksum) > 0 {
+	if len(incomingSoftwareByChecksum) > 0 {
 		// Pre-insert software and titles in small batches
-		err = ds.preInsertSoftwareInventory(ctx, existingSoftware, incomingByChecksum, existingTitlesForNewSoftware)
+		err = ds.preInsertSoftwareInventory(ctx, existingSoftwareSummaries, incomingSoftwareByChecksum, incomingChecksumsToExistingTitles)
 		if err != nil {
 			return nil, ctxerr.Wrap(ctx, err, "pre-insert software inventory")
 		}
@@ -436,7 +438,7 @@ func (ds *Datastore) applyChangesForNewSoftwareDB(
 
 			// Link the pre-inserted software to this host
 			// Software inventory entries were already created in Phase 1
-			inserted, err := ds.linkSoftwareToHost(ctx, tx, hostID, incomingByChecksum)
+			inserted, err := ds.linkSoftwareToHost(ctx, tx, hostID, incomingSoftwareByChecksum)
 			if err != nil {
 				return err
 			}
@@ -533,61 +535,66 @@ func checkForDeletedInstalledSoftware(ctx context.Context, tx sqlx.ExtContext, d
 func (ds *Datastore) getExistingSoftware(
 	ctx context.Context, current map[string]fleet.Software, incoming map[string]fleet.Software,
 ) (
-	currentSoftware []softwareIDChecksum,
-	incomingChecksumToSoftware map[string]fleet.Software,
-	incomingChecksumToTitle map[string]fleet.SoftwareTitle,
+	currentSoftwareSummaries []softwareSummary,
+	newChecksumsToSoftware map[string]fleet.Software,
+	incomingChecksumsToTitles map[string]fleet.SoftwareTitle,
 	err error,
 ) {
+	// TODO(jacob) - the `incoming` argument here should already contain a map of checksum:Software, put
+	// together by the `nothingChanged` function upstream. Is this redundant?
 	// Compute checksums for all incoming software, which we will use for faster retrieval, since checksum is a unique index
-	incomingChecksumToSoftware = make(map[string]fleet.Software, len(current))
-	newSoftware := make(map[string]struct{})
+	newChecksumsToSoftware = make(map[string]fleet.Software, len(current))
+	// TODO(jacob) - below set seems to be the same as above map but without Software values for each key.
+	// Are both necessary, or can we just use the map everywhere?
+	setOfNewSWChecksums := make(map[string]struct{})
 	for uniqueName, s := range incoming {
 		_, ok := current[uniqueName]
 		if !ok {
+			// -> incoming SW is new
 			checksum, err := s.ComputeRawChecksum()
 			if err != nil {
 				return nil, nil, nil, err
 			}
-			incomingChecksumToSoftware[string(checksum)] = s
-			newSoftware[string(checksum)] = struct{}{}
+			newChecksumsToSoftware[string(checksum)] = s
+			setOfNewSWChecksums[string(checksum)] = struct{}{}
 		}
 	}
 
-	if len(incomingChecksumToSoftware) > 0 {
-		keys := make([]string, 0, len(incomingChecksumToSoftware))
-		for checksum := range incomingChecksumToSoftware {
-			keys = append(keys, checksum)
+	if len(newChecksumsToSoftware) > 0 {
+		sliceOfNewSWChecksums := make([]string, 0, len(newChecksumsToSoftware))
+		for checksum := range newChecksumsToSoftware {
+			sliceOfNewSWChecksums = append(sliceOfNewSWChecksums, checksum)
 		}
-		// We use the replica DB for retrieval to minimize the traffic to the master DB.
-		// It is OK if the software is not found in the replica DB, because we will then attempt to insert it in the master DB.
-		currentSoftware, err = getSoftwareIDsByChecksums(ctx, ds.reader(ctx), keys)
+		// We use the replica DB for retrieval to minimize the traffic to the writer DB.
+		// It is OK if the software is not found in the replica DB, because we will then attempt to insert it in the writer DB.
+		currentSoftwareSummaries, err = getExistingSoftwareSummariesByChecksums(ctx, ds.reader(ctx), sliceOfNewSWChecksums)
 		if err != nil {
 			return nil, nil, nil, err
 		}
 
-		for _, currentSoftwareItem := range currentSoftware {
-			_, ok := incomingChecksumToSoftware[currentSoftwareItem.Checksum]
+		for _, currentSoftwareSummary := range currentSoftwareSummaries {
+			_, ok := newChecksumsToSoftware[currentSoftwareSummary.Checksum]
 			if !ok {
 				// This should never happen. If it does, we have a bug.
 				return nil, nil, nil, ctxerr.New(
-					ctx, fmt.Sprintf("current software: software not found for checksum %s", hex.EncodeToString([]byte(currentSoftwareItem.Checksum))),
+					ctx, fmt.Sprintf("current software: software not found for checksum %s", hex.EncodeToString([]byte(currentSoftwareSummary.Checksum))),
 				)
 			}
-			delete(newSoftware, currentSoftwareItem.Checksum)
+			delete(setOfNewSWChecksums, currentSoftwareSummary.Checksum)
 		}
 	}
 
-	if len(newSoftware) == 0 {
-		return currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, nil
+	if len(setOfNewSWChecksums) == 0 {
+		return currentSoftwareSummaries, newChecksumsToSoftware, incomingChecksumsToTitles, nil
 	}
 
 	// There's new software, so we try to get the titles already stored in `software_titles` for them.
-	incomingChecksumToTitle, _, err = ds.getIncomingSoftwareChecksumsToExistingTitles(ctx, newSoftware, incomingChecksumToSoftware)
+	incomingChecksumsToTitles, _, err = ds.getIncomingSoftwareChecksumsToExistingTitles(ctx, setOfNewSWChecksums, newChecksumsToSoftware)
 	if err != nil {
 		return nil, nil, nil, ctxerr.Wrap(ctx, err, "get incoming software checksums to existing titles")
 	}
 
-	return currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, nil
+	return currentSoftwareSummaries, newChecksumsToSoftware, incomingChecksumsToTitles, nil
 }
 
 // getIncomingSoftwareChecksumsToExistingTitles loads the existing titles for the new incoming software.
@@ -596,13 +603,16 @@ func (ds *Datastore) getExistingSoftware(
 // To make best use of separate indexes, it runs two queries to get the existing titles from the DB:
 //   - One query for software with bundle_identifier.
 //   - One query for software without bundle_identifier.
+//
+// TODO(jacob) - consider index and appropriate query here for Windows software `upgrade_code`s, similar to
+// bundle identifier, if needed for optimization
 func (ds *Datastore) getIncomingSoftwareChecksumsToExistingTitles(
 	ctx context.Context,
 	newSoftwareChecksums map[string]struct{},
 	incomingChecksumToSoftware map[string]fleet.Software,
 ) (map[string]fleet.SoftwareTitle, map[string]fleet.Software, error) {
 	var (
-		incomingChecksumToTitle     = make(map[string]fleet.SoftwareTitle, len(newSoftwareChecksums))
+		incomingChecksumsToTitles   = make(map[string]fleet.SoftwareTitle, len(newSoftwareChecksums))
 		argsWithoutBundleIdentifier []any
 		argsWithBundleIdentifier    []any
 		uniqueTitleStrToChecksums   = make(map[string][]string)
@@ -614,6 +624,7 @@ func (ds *Datastore) getIncomingSoftwareChecksumsToExistingTitles(
 			bundleIDsToIncomingNames[sw.BundleIdentifier] = sw.Name
 			argsWithBundleIdentifier = append(argsWithBundleIdentifier, sw.BundleIdentifier)
 		} else {
+			// TODO(jacob) - consider `upgrade_code` here and below if needed for additional specificity
 			argsWithoutBundleIdentifier = append(argsWithoutBundleIdentifier, sw.Name, sw.Source, sw.ExtensionFor)
 		}
 		// Map software title identifier to software checksums so that we can map checksums to actual titles later.
@@ -668,7 +679,7 @@ func (ds *Datastore) getIncomingSoftwareChecksumsToExistingTitles(
 			if ok {
 				// Map all checksums that correspond to this title
 				for _, checksum := range checksums {
-					incomingChecksumToTitle[checksum] = title
+					incomingChecksumsToTitles[checksum] = title
 				}
 			}
 		}
@@ -678,11 +689,13 @@ func (ds *Datastore) getIncomingSoftwareChecksumsToExistingTitles(
 	existingBundleIDsToUpdate := make(map[string]fleet.Software)
 	if len(argsWithBundleIdentifier) > 0 {
 		// no-op code change
-		incomingChecksumToTitle = make(map[string]fleet.SoftwareTitle, len(newSoftwareChecksums))
+		// TODO(jacob) - this var name is shadowing the one in the outer scope. Is this successfully
+		// adding titles-by-checksum for software with bundle ids?
+		incomingChecksumsToTitles = make(map[string]fleet.SoftwareTitle, len(newSoftwareChecksums))
 		stmtBundleIdentifier := `SELECT id, name, source, extension_for, bundle_identifier FROM software_titles WHERE bundle_identifier IN (?)`
 		stmtBundleIdentifier, argsWithBundleIdentifier, err := sqlx.In(stmtBundleIdentifier, argsWithBundleIdentifier)
 		if err != nil {
-			return nil, nil, ctxerr.Wrap(ctx, err, "build query to existing titles with bundle_identifier")
+			return nil, nil, ctxerr.Wrap(ctx, err, "build query to get existing titles with bundle_identifier")
 		}
 		var existingSoftwareTitlesForNewSoftwareWithBundleIdentifier []fleet.SoftwareTitle
 		if err := sqlx.SelectContext(ctx, ds.reader(ctx), &existingSoftwareTitlesForNewSoftwareWithBundleIdentifier, stmtBundleIdentifier, argsWithBundleIdentifier...); err != nil {
@@ -696,13 +709,13 @@ func (ds *Datastore) getIncomingSoftwareChecksumsToExistingTitles(
 			if withoutName {
 				// Map all checksums that correspond to this title
 				for _, checksum := range checksums {
-					incomingChecksumToTitle[checksum] = title
+					incomingChecksumsToTitles[checksum] = title
 				}
 			}
 		}
 	}
 
-	return incomingChecksumToTitle, existingBundleIDsToUpdate, nil
+	return incomingChecksumsToTitles, existingBundleIDsToUpdate, nil
 }
 
 // BundleIdentifierOrName returns the bundle identifier if it is not empty, otherwise name
@@ -783,9 +796,9 @@ func longestCommonPrefix(strs []string) string {
 // to reduce lock contention. These operations are idempotent due to INSERT IGNORE.
 func (ds *Datastore) preInsertSoftwareInventory(
 	ctx context.Context,
-	existingSoftware []softwareIDChecksum,
-	softwareChecksums map[string]fleet.Software,
-	existingTitlesForNewSoftware map[string]fleet.SoftwareTitle,
+	existingSoftwareSummaries []softwareSummary,
+	incomingSoftwareByChecksum map[string]fleet.Software,
+	incomingChecksumsToExistingTitles map[string]fleet.SoftwareTitle,
 ) error {
 	type titleKey struct {
 		name         string
@@ -798,14 +811,14 @@ func (ds *Datastore) preInsertSoftwareInventory(
 	// Collect all software that needs to be inserted
 	needsInsert := make(map[string]fleet.Software)
 	bundleGroups := make(map[titleKey][]string)
-	keys := make([]string, 0, len(softwareChecksums))
+	keys := make([]string, 0, len(incomingSoftwareByChecksum))
 
-	existingSet := make(map[string]struct{}, len(existingSoftware))
-	for _, es := range existingSoftware {
+	existingSet := make(map[string]struct{}, len(existingSoftwareSummaries))
+	for _, es := range existingSoftwareSummaries {
 		existingSet[es.Checksum] = struct{}{}
 	}
 
-	for checksum, sw := range softwareChecksums {
+	for checksum, sw := range incomingSoftwareByChecksum {
 		if _, ok := existingSet[checksum]; !ok {
 			needsInsert[checksum] = sw
 			keys = append(keys, checksum)
@@ -861,7 +874,7 @@ func (ds *Datastore) preInsertSoftwareInventory(
 			// First insert any needed software titles
 			newTitlesNeeded := make(map[string]fleet.SoftwareTitle)
 			for checksum, sw := range batchSoftware {
-				if _, ok := existingTitlesForNewSoftware[checksum]; !ok {
+				if _, ok := incomingChecksumsToExistingTitles[checksum]; !ok {
 					titleName := sw.Name
 					if sw.BundleIdentifier != "" {
 						key := titleKey{
@@ -886,17 +899,23 @@ func (ds *Datastore) preInsertSoftwareInventory(
 					if sw.ApplicationID != nil && *sw.ApplicationID != "" {
 						st.ApplicationID = sw.ApplicationID
 					}
+					if sw.UpgradeCode != nil {
+						// intentionally write both empty and non-empty strings as upgrade codes
+						st.UpgradeCode = sw.UpgradeCode
+					}
 					newTitlesNeeded[checksum] = st
 				}
 			}
 
 			// Map to store title IDs for all titles (both existing and new)
-			titleIDsByChecksum := make(map[string]uint, len(existingTitlesForNewSoftware))
+			titleIDsByChecksum := make(map[string]uint, len(incomingChecksumsToExistingTitles))
 
 			// First, add existing titles to the map
-			for checksum, title := range existingTitlesForNewSoftware {
+			for checksum, title := range incomingChecksumsToExistingTitles {
 				titleIDsByChecksum[checksum] = title.ID
 			}
+			// TODO: somewhere around here: if new SW title has diff upgrade_code from existing, log as an error and
+			// do NOT insert the new title
 
 			if len(newTitlesNeeded) > 0 {
 				uniqueTitlesToInsert := make(map[titleKey]fleet.SoftwareTitle)
@@ -919,19 +938,20 @@ func (ds *Datastore) preInsertSoftwareInventory(
 				}
 
 				// Insert software titles
-				const numberOfArgsPerSoftwareTitles = 6
-				titlesValues := strings.TrimSuffix(strings.Repeat("(?,?,?,?,?,?),", len(uniqueTitlesToInsert)), ",")
-				titlesStmt := fmt.Sprintf("INSERT IGNORE INTO software_titles (name, source, extension_for, bundle_identifier, is_kernel, application_id) VALUES %s", titlesValues)
+				const numberOfArgsPerSoftwareTitles = 7
+				titlesValues := strings.TrimSuffix(strings.Repeat("(?,?,?,?,?,?,?),", len(uniqueTitlesToInsert)), ",")
+				titlesStmt := fmt.Sprintf("INSERT IGNORE INTO software_titles (name, source, extension_for, bundle_identifier, is_kernel, application_id, upgrade_code) VALUES %s", titlesValues)
 				titlesArgs := make([]any, 0, len(uniqueTitlesToInsert)*numberOfArgsPerSoftwareTitles)
 
 				for _, title := range uniqueTitlesToInsert {
-					titlesArgs = append(titlesArgs, title.Name, title.Source, title.ExtensionFor, title.BundleIdentifier, title.IsKernel, title.ApplicationID)
+					titlesArgs = append(titlesArgs, title.Name, title.Source, title.ExtensionFor, title.BundleIdentifier, title.IsKernel, title.ApplicationID, title.UpgradeCode)
 				}
 
 				if _, err := tx.ExecContext(ctx, titlesStmt, titlesArgs...); err != nil {
 					return ctxerr.Wrap(ctx, err, "pre-insert software_titles")
 				}
 
+				// TODO(jacob) - incorporate UpgradeCode here?
 				// Retrieve the IDs for the titles we just inserted (or that already existed)
 				var titlesData []struct {
 					ID               uint    `db:"id"`
@@ -993,9 +1013,9 @@ func (ds *Datastore) preInsertSoftwareInventory(
 			}
 
 			// Insert software entries
-			const numberOfArgsPerSoftware = 12
+			const numberOfArgsPerSoftware = 13
 			values := strings.TrimSuffix(
-				strings.Repeat("(?,?,?,?,?,?,?,?,?,?,?,?),", len(batchKeys)), ",",
+				strings.Repeat("(?,?,?,?,?,?,?,?,?,?,?,?,?),", len(batchKeys)), ",",
 			)
 			stmt := fmt.Sprintf(
 				`INSERT IGNORE INTO software (
@@ -1010,7 +1030,8 @@ func (ds *Datastore) preInsertSoftwareInventory(
 					extension_for,
 					title_id,
 					checksum,
-					application_id
+					application_id,
+					upgrade_code
 				) VALUES %s`,
 				values,
 			)
@@ -1030,7 +1051,7 @@ func (ds *Datastore) preInsertSoftwareInventory(
 				}
 				args = append(
 					args, sw.Name, sw.Version, sw.Source, sw.Release, sw.Vendor, sw.Arch,
-					sw.BundleIdentifier, sw.ExtensionID, sw.ExtensionFor, titleID, checksum, sw.ApplicationID,
+					sw.BundleIdentifier, sw.ExtensionID, sw.ExtensionFor, titleID, checksum, sw.ApplicationID, sw.UpgradeCode,
 				)
 			}
 
@@ -1079,20 +1100,20 @@ func (ds *Datastore) linkSoftwareToHost(
 	// Get all software IDs (they should exist from pre-insertion).
 	// This ensures that we're not creating orphaned references (where software was deleted between pre-insertion and now).
 	// This DB call could be removed to squeeze our a little more performance at the risk of orphaned references.
-	allSoftware, err := getSoftwareIDsByChecksums(ctx, tx, allChecksums)
+	allSoftwareSummaries, err := getExistingSoftwareSummariesByChecksums(ctx, tx, allChecksums)
 	if err != nil {
 		return nil, err
 	}
 
 	// Build ID map
-	softwareByChecksum := make(map[string]softwareIDChecksum)
-	for _, s := range allSoftware {
-		softwareByChecksum[s.Checksum] = s
+	softwareSummaryByChecksum := make(map[string]softwareSummary)
+	for _, s := range allSoftwareSummaries {
+		softwareSummaryByChecksum[s.Checksum] = s
 	}
 
 	// Link software to host
 	for checksum, sw := range softwareChecksums {
-		if existing, ok := softwareByChecksum[checksum]; ok {
+		if existing, ok := softwareSummaryByChecksum[checksum]; ok {
 			sw.ID = existing.ID
 			insertsHostSoftware = append(insertsHostSoftware, hostID, sw.ID, sw.LastOpenedAt)
 			insertedSoftware = append(insertedSoftware, sw)
@@ -1120,21 +1141,20 @@ func (ds *Datastore) linkSoftwareToHost(
 	return insertedSoftware, nil
 }
 
-func getSoftwareIDsByChecksums(ctx context.Context, tx sqlx.QueryerContext, checksums []string) ([]softwareIDChecksum, error) {
+func getExistingSoftwareSummariesByChecksums(ctx context.Context, tx sqlx.QueryerContext, checksums []string) ([]softwareSummary, error) {
 	if len(checksums) == 0 {
-		return []softwareIDChecksum{}, nil
+		return []softwareSummary{}, nil
 	}
 
-	// get existing software ids for checksums
-	stmt, args, err := sqlx.In("SELECT name, id, checksum, title_id, bundle_identifier, source FROM software WHERE checksum IN (?)", checksums)
+	stmt, args, err := sqlx.In("SELECT name, id, checksum, title_id, bundle_identifier, source, upgrade_code FROM software WHERE checksum IN (?)", checksums)
 	if err != nil {
-		return nil, ctxerr.Wrap(ctx, err, "build select software query")
+		return nil, ctxerr.Wrap(ctx, err, "build select software summaries query")
 	}
-	var existingSoftware []softwareIDChecksum
-	if err = sqlx.SelectContext(ctx, tx, &existingSoftware, stmt, args...); err != nil {
-		return nil, ctxerr.Wrap(ctx, err, "get existing software")
+	var existingSoftwareSummaries []softwareSummary
+	if err = sqlx.SelectContext(ctx, tx, &existingSoftwareSummaries, stmt, args...); err != nil {
+		return nil, ctxerr.Wrap(ctx, err, "get existing software summaries")
 	}
-	return existingSoftware, nil
+	return existingSoftwareSummaries, nil
 }
 
 // update host_software when incoming software has a significantly more recent
@@ -1330,6 +1350,7 @@ func selectSoftwareSQL(opts fleet.SoftwareListOptions) (string, []interface{}, e
 			"s.arch",
 			"s.application_id",
 			"s.title_id",
+			"s.upgrade_code",
 			goqu.I("scp.cpe").As("generated_cpe"),
 		).
 		// Include this in the sub-query in case we want to sort by 'generated_cpe'
@@ -1520,6 +1541,7 @@ func selectSoftwareSQL(opts fleet.SoftwareListOptions) (string, []interface{}, e
 			"s.arch",
 			"s.application_id",
 			"s.title_id",
+			"s.upgrade_code",
 			goqu.COALESCE(goqu.I("s.generated_cpe"), "").As("generated_cpe"),
 			"scv.cve",
 			"scv.created_at",
@@ -1669,7 +1691,7 @@ func (ds *Datastore) AllSoftwareIterator(
 	stmt := `SELECT
 		s.id, s.name, s.version, s.source, s.bundle_identifier, s.release, s.arch, s.vendor, s.extension_for, s.extension_id, s.title_id,
 		COALESCE(sc.cpe, '') AS generated_cpe
-	FROM software s
+		FROM software s
 	LEFT JOIN software_cpe sc ON (s.id=sc.software_id)`
 
 	var conditionals []string
@@ -1872,6 +1894,7 @@ func (ds *Datastore) SoftwareByID(ctx context.Context, id uint, teamID *uint, in
 			"s.source",
 			"s.extension_for",
 			"s.bundle_identifier",
+			"s.upgrade_code",
 			"s.release",
 			"s.vendor",
 			"s.arch",
@@ -2499,6 +2522,7 @@ func (ds *Datastore) ListCVEs(ctx context.Context, maxAge time.Duration) ([]flee
 	return result, nil
 }
 
+// TODO(jacob) SoftwareUpgradeCode ? SoftwareUpgradeCodeList ?
 type hostSoftware struct {
 	fleet.HostSoftwareWithInstaller
 
@@ -2546,9 +2570,11 @@ type hostSoftware struct {
 	InHouseAppPlatformList    *string `db:"in_house_app_platform_list"`
 	InHouseAppVersionList     *string `db:"in_house_app_version_list"`
 	InHouseAppSelfServiceList *string `db:"in_house_app_self_service_list"`
+	SoftwareUpgradeCodeList   *string `db:"software_upgrade_code_list"`
 }
 
 func hostInstalledSoftware(ds *Datastore, ctx context.Context, hostID uint) ([]*hostSoftware, error) {
+	// TODO(jacob)?: software_titles.upgrade_code AS upgrade_code,
 	installedSoftwareStmt := `
 		SELECT
 			software_titles.id AS id,
@@ -3896,6 +3922,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt
 				st.name,
 				st.source,
 				st.extension_for,
+				st.upgrade_code,
 				si.id as installer_id,
 				si.self_service as package_self_service,
 				si.filename as package_name,
@@ -4826,6 +4853,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt
 					software_titles.name,
 					software_titles.source AS source,
 					software_titles.extension_for AS extension_for,
+					software_titles.upgrade_code AS upgrade_code, -- should be empty or non-empty string for "programs" sourced software, null otherwise
 					software_installers.id AS installer_id,
 					software_installers.self_service AS package_self_service,
 					software_installers.filename AS package_name,
@@ -4834,6 +4862,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt
 					GROUP_CONCAT(software.id) AS software_id_list,
 					GROUP_CONCAT(software.source) AS software_source_list,
 					GROUP_CONCAT(software.extension_for) AS software_extension_for_list,
+					GROUP_CONCAT(software.upgrade_code) AS software_upgrade_code_list,
 					GROUP_CONCAT(software.version) AS version_list,
 					GROUP_CONCAT(software.bundle_identifier) AS bundle_identifier_list,
 					NULL AS vpp_app_adam_id_list,
@@ -4852,6 +4881,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt
 					software_titles.name,
 					software_titles.source,
 					software_titles.extension_for,
+					software_titles.upgrade_code,
 					software_installers.id,
 					software_installers.self_service,
 					software_installers.filename,
@@ -4868,6 +4898,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt
 					software_titles.name,
 					software_titles.source AS source,
 					software_titles.extension_for AS extension_for,
+					software_titles.upgrade_code AS upgrade_code, -- should always be null for vpp (mac) apps
 					NULL AS installer_id,
 					NULL AS package_self_service,
 					NULL AS package_name,
@@ -4876,6 +4907,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt
 					NULL AS software_id_list,
 					NULL AS software_source_list,
 					NULL AS software_extension_for_list,
+					NULL AS software_upgrade_code_list,
 					NULL AS version_list,
 					NULL AS bundle_identifier_list,
 					GROUP_CONCAT(vpp_apps.adam_id) AS vpp_app_adam_id_list,
@@ -4893,7 +4925,8 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt
 					software_titles.id,
 					software_titles.name,
 					software_titles.source,
-					software_titles.extension_for
+					software_titles.extension_for,
+					software_titles.upgrade_code
 			`)
 		}
 
@@ -4906,6 +4939,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt
 					software_titles.name,
 					software_titles.source AS source,
 					software_titles.extension_for AS extension_for,
+					software_titles.upgrade_code AS upgrade_code,
 					NULL AS installer_id,
 					NULL AS package_self_service,
 					NULL AS package_name,
@@ -4914,6 +4948,7 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt
 					NULL AS software_id_list,
 					NULL AS software_source_list,
 					NULL AS software_extension_for_list,
+					NULL AS software_upgrade_code_list,
 					NULL AS version_list,
 					NULL AS bundle_identifier_list,
 					NULL AS vpp_app_adam_id_list,
@@ -4931,7 +4966,8 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt
 					software_titles.id,
 					software_titles.name,
 					software_titles.source,
-					software_titles.extension_for
+					software_titles.extension_for,
+					software_titles.upgrade_code
 			`)
 		}
 		stmt = fmt.Sprintf(stmt, replacements...)
@@ -5457,6 +5493,7 @@ SELECT
 	st.name,
 	st.source,
 	st.extension_for,
+	st.upgrade_code,
 	st.bundle_identifier,
 	0 as vpp_apps_count
 FROM software_titles st
@@ -5471,6 +5508,7 @@ SELECT
 	st.name,
 	st.source,
 	st.extension_for,
+	st.upgrade_code,
 	st.bundle_identifier,
 	1 as vpp_apps_count
 FROM software_titles st

server/datastore/mysql/software_installers_test.go

@@ -3619,7 +3619,6 @@ func testSoftwareTitleDisplayName(t *testing.T, ds *Datastore) {
 	require.NoError(t, err)
 	assert.Equal(t, titleID, *software.TitleID)
 	assert.Empty(t, software.DisplayName)
-
 }
 
 func testMatchOrCreateSoftwareInstallerDuplicateHash(t *testing.T, ds *Datastore) {

server/datastore/mysql/software_test.go

@@ -3707,6 +3707,8 @@ func testVerifySoftwareChecksum(t *testing.T, ds *Datastore) {
 		{Name: "foo", Version: "0.0.1", Source: "test", ExtensionID: "ext"},
 		{Name: "foo", Version: "0.0.2", Source: "test"},
 		{Name: "foo", Version: "0.0.2", Source: "test", ApplicationID: ptr.String("foo.bar.baz")},
+		{Name: "foo", Version: "0.0.2", Source: "programs", UpgradeCode: ptr.String("{55ac7218-24cb-4b99-9449-f28d9c59cc7e}")},
+		{Name: "foo", Version: "0.0.2", Source: "programs", UpgradeCode: ptr.String("")},
 	}
 
 	_, err := ds.UpdateHostSoftware(ctx, host.ID, software)
@@ -3722,7 +3724,7 @@ func testVerifySoftwareChecksum(t *testing.T, ds *Datastore) {
 		var got fleet.Software
 		ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
 			return sqlx.GetContext(ctx, q, &got,
-				`SELECT name, version, source, bundle_identifier, `+"`release`"+`, arch, vendor, extension_for, extension_id, application_id FROM software WHERE checksum = UNHEX(?)`, cs)
+				`SELECT name, version, source, bundle_identifier, `+"`release`"+`, arch, vendor, extension_for, extension_id, application_id, upgrade_code FROM software WHERE checksum = UNHEX(?)`, cs)
 		})
 		require.Equal(t, software[i], got)
 	}
@@ -9157,9 +9159,9 @@ func testCheckForDeletedInstalledSoftware(t *testing.T, ds *Datastore) {
 	require.NoError(t, err)
 	require.NoError(t, ds.AddHostsToTeam(ctx, fleet.NewAddHostsToTeamParams(&team1.ID, []uint{host1.ID})))
 
-	existingSw, err := fleet.SoftwareFromOsqueryRow("htop", "3.4.0-2", "deb_packages", "", "", "", "", "", "", "", "")
+	existingSw, err := fleet.SoftwareFromOsqueryRow("htop", "3.4.0-2", "deb_packages", "", "", "", "", "", "", "", "", "")
 	require.NoError(t, err)
-	updateSw, err := fleet.SoftwareFromOsqueryRow("htop", "3.4.1-5", "deb_packages", "", "", "", "", "", "", "", "")
+	updateSw, err := fleet.SoftwareFromOsqueryRow("htop", "3.4.1-5", "deb_packages", "", "", "", "", "", "", "", "", "")
 	require.NoError(t, err)
 
 	_, err = ds.UpdateHostSoftware(ctx, host1.ID, []fleet.Software{*existingSw})

server/datastore/mysql/software_titles.go

@@ -48,6 +48,7 @@ SELECT
 	st.extension_for,
 	st.bundle_identifier,
 	st.application_id,
+	st.upgrade_code,
 	COALESCE(sthc.hosts_count, 0) AS hosts_count,
 	MAX(sthc.updated_at) AS counts_updated_at,
 	COUNT(si.id) as software_installers_count,
@@ -419,6 +420,7 @@ SELECT
 	,st.extension_for
 	,st.bundle_identifier
 	,st.application_id
+	,st.upgrade_code
 	,MAX(COALESCE(sthc.hosts_count, 0)) as hosts_count
 	,MAX(COALESCE(sthc.updated_at, date('0001-01-01 00:00:00'))) as counts_updated_at
 	{{if hasTeamID .}}

server/datastore/mysql/software_titles_test.go

@@ -2217,7 +2217,7 @@ func testSoftwareTitleHostCount(t *testing.T, ds *Datastore) {
 	}
 
 	// install software on host
-	updateSw, err := fleet.SoftwareFromOsqueryRow("foo", "1.0", "apps", "", "", "", "", "com.foo.installer", "", "", "")
+	updateSw, err := fleet.SoftwareFromOsqueryRow("foo", "1.0", "apps", "", "", "", "", "com.foo.installer", "", "", "", "")
 	require.NoError(t, err)
 
 	hostInstall1, err := ds.InsertSoftwareInstallRequest(ctx, host1.ID, installers[0], fleet.HostSoftwareInstallOptions{})