Adding name to software checksum for mac software (#34097)

**Related issue:** Resolves #28788

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.
- [x] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements)

## Testing

- [x] Added/updated automated tests
- [x] Where appropriate, [automated tests simulate multiple hosts and
test for host
isolation](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/reference/patterns-backend.md#unit-testing)
(updates to one hosts's records do not affect another)
- [x] QA'd all new/changed functionality manually

## Database migrations

- [x] Checked schema for all modified table for columns that will
auto-update timestamps during migration.
- [x] Confirmed that updating the timestamps is acceptable, and will not
cause unwanted side effects.
- [x] Ensured the correct collation is explicitly set for character
columns (`COLLATE utf8mb4_unicode_ci`).

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Bug Fixes**
* macOS app checksums now include the app name, improving grouping,
deduplication, and preventing mis-linking or duplicate entries when
multiple names share a bundle ID.
* More stable title handling when bundle IDs are missing, reducing
unintended renames and mismatches.

* **Tests**
* Re-enabled related host-software tests and added a
longest-common-prefix test to validate name reconciliation.

* **Chores**
* Database migration added to recalculate checksums for affected macOS
app records.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Konstantin Sykulev
2025-10-14 17:36:34 -05:00
committed by GitHub
parent 0e9bba4774
commit bb4b62bd0f
7 changed files with 507 additions and 383 deletions
+1
View File
@@ -0,0 +1 @@
* Added software name into checksum calculation for macos apps
@@ -0,0 +1,70 @@
package tables
import (
"database/sql"
"fmt"
)
func init() {
MigrationClient.AddMigration(Up_20251010153829, Down_20251010153829)
}
func Up_20251010153829(tx *sql.Tx) error {
var minID, maxID sql.NullInt64
err := tx.QueryRow(`
SELECT MIN(id), MAX(id)
FROM software
WHERE source = 'apps'
AND bundle_identifier IS NOT NULL
AND bundle_identifier != ''
`).Scan(&minID, &maxID)
if err != nil {
return fmt.Errorf("getting ID range: %w", err)
}
if !minID.Valid || !maxID.Valid {
return nil
}
const batchSize = 10000
for startID := minID.Int64; startID <= maxID.Int64; startID += batchSize {
endID := startID + batchSize - 1
if endID > maxID.Int64 {
endID = maxID.Int64
}
softwareStmt := `
UPDATE software SET
checksum = UNHEX(
MD5(
-- concatenate with separator \x00
CONCAT_WS(CHAR(0),
version,
source,
bundle_identifier,
` + "`release`" + `,
arch,
vendor,
extension_for,
extension_id,
name
)
)
)
WHERE source = 'apps'
AND bundle_identifier IS NOT NULL
AND bundle_identifier != ''
AND id >= ? AND id <= ?
`
_, err = tx.Exec(softwareStmt, startID, endID)
if err != nil {
return fmt.Errorf("updating software checksums (batch %d-%d): %w", startID, endID, err)
}
}
return nil
}
func Down_20251010153829(tx *sql.Tx) error {
return nil
}
@@ -0,0 +1,122 @@
package tables
import (
"crypto/md5" //nolint:gosec // MD5 is used for checksums, not security
"database/sql"
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestUp_20251010153829(t *testing.T) {
db := applyUpToPrev(t)
computeOldChecksum := func(name, version, source, bundleID, release, arch, vendor, extensionFor, extensionID string) []byte {
h := md5.New() //nolint:gosec
cols := []string{version, source, bundleID, release, arch, vendor, extensionFor, extensionID}
if source != "apps" {
cols = append([]string{name}, cols...)
}
_, _ = fmt.Fprint(h, strings.Join(cols, "\x00"))
return h.Sum(nil)
}
computeNewChecksum := func(name, version, source, bundleID, release, arch, vendor, extensionFor, extensionID string) []byte {
h := md5.New() //nolint:gosec
cols := []string{version, source, bundleID, release, arch, vendor, extensionFor, extensionID, name}
_, _ = fmt.Fprint(h, strings.Join(cols, "\x00"))
return h.Sum(nil)
}
insertTitle := `INSERT INTO software_titles (name, source, extension_for, bundle_identifier) VALUES (?, ?, ?, ?)`
result, err := db.Exec(insertTitle, "Test App", "apps", "", "com.test.app")
require.NoError(t, err)
titleID, err := result.LastInsertId()
require.NoError(t, err)
insertSoftware := `INSERT INTO software
(name, version, source, bundle_identifier, ` + "`release`" + `, arch, vendor, extension_for, extension_id, checksum, title_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
// software with bundle_identifier should be updated
app1Name := "GoLand.app"
app1BundleID := "com.jetbrains.goland"
app1OldChecksum := computeOldChecksum(app1Name, "2023.1", "apps", app1BundleID, "", "x86_64", "JetBrains", "", "")
app1NewChecksum := computeNewChecksum(app1Name, "2023.1", "apps", app1BundleID, "", "x86_64", "JetBrains", "", "")
_, err = db.Exec(insertSoftware, app1Name, "2023.1", "apps", app1BundleID, "", "x86_64", "JetBrains", "", "", app1OldChecksum, titleID)
require.NoError(t, err)
app2Name := "GoLand 2.app"
app2BundleID := "com.jetbrains.goland"
app2OldChecksum := computeOldChecksum(app2Name, "2023.2", "apps", app2BundleID, "", "x86_64", "JetBrains", "", "")
app2NewChecksum := computeNewChecksum(app2Name, "2023.2", "apps", app2BundleID, "", "x86_64", "JetBrains", "", "")
_, err = db.Exec(insertSoftware, app2Name, "2023.2", "apps", app2BundleID, "", "x86_64", "JetBrains", "", "", app2OldChecksum, titleID)
require.NoError(t, err)
// softwares without bundle_identifier - no update
app3Name := "SomeApp.app"
app3OldChecksum := computeOldChecksum(app3Name, "1.0", "apps", "", "", "x86_64", "Vendor", "", "")
_, err = db.Exec(insertSoftware, app3Name, "1.0", "apps", nil, "", "x86_64", "Vendor", "", "", app3OldChecksum, titleID)
require.NoError(t, err)
app4Name := "AnotherApp.app"
app4OldChecksum := computeOldChecksum(app4Name, "2.0", "apps", "", "", "arm64", "Another Vendor", "", "")
_, err = db.Exec(insertSoftware, app4Name, "2.0", "apps", "", "", "arm64", "Another Vendor", "", "", app4OldChecksum, titleID)
require.NoError(t, err)
// Windows software - no update
winName := "Notepad++"
winOldChecksum := computeOldChecksum(winName, "8.5.0", "programs", "", "", "x86_64", "Don Ho", "", "")
_, err = db.Exec(insertSoftware, winName, "8.5.0", "programs", nil, "", "x86_64", "Don Ho", "", "", winOldChecksum, titleID)
require.NoError(t, err)
// Linux software - no update
linuxName := "vim"
linuxOldChecksum := computeOldChecksum(linuxName, "8.2", "deb_packages", "", "1ubuntu1", "amd64", "Ubuntu", "", "")
_, err = db.Exec(insertSoftware, linuxName, "8.2", "deb_packages", nil, "1ubuntu1", "amd64", "Ubuntu", "", "", linuxOldChecksum, titleID)
require.NoError(t, err)
applyNext(t, db)
type softwareRow struct {
Name string `db:"name"`
Source string `db:"source"`
BundleIdentifier sql.NullString `db:"bundle_identifier"`
Checksum []byte `db:"checksum"`
}
var software []softwareRow
err = db.Select(&software, `SELECT name, source, bundle_identifier, checksum FROM software ORDER BY name`)
require.NoError(t, err)
require.Len(t, software, 6)
for _, sw := range software {
switch sw.Name {
case app1Name:
require.Equal(t, app1NewChecksum, sw.Checksum)
require.True(t, sw.BundleIdentifier.Valid)
require.Equal(t, app1BundleID, sw.BundleIdentifier.String)
case app2Name:
require.Equal(t, app2NewChecksum, sw.Checksum)
require.True(t, sw.BundleIdentifier.Valid)
require.Equal(t, app2BundleID, sw.BundleIdentifier.String)
case app3Name:
require.Equal(t, app3OldChecksum, sw.Checksum)
require.False(t, sw.BundleIdentifier.Valid)
case app4Name:
require.Equal(t, app4OldChecksum, sw.Checksum)
require.True(t, sw.BundleIdentifier.Valid)
require.Equal(t, "", sw.BundleIdentifier.String)
case winName:
require.Equal(t, winOldChecksum, sw.Checksum)
require.Equal(t, "programs", sw.Source)
case linuxName:
require.Equal(t, linuxOldChecksum, sw.Checksum)
require.Equal(t, "deb_packages", sw.Source)
default:
t.Fatalf("Unexpected software entry: %s", sw.Name)
}
}
}
File diff suppressed because one or more lines are too long
+132 -287
View File
@@ -6,6 +6,7 @@ import (
"encoding/hex"
"errors"
"fmt"
"regexp"
"sort"
"strconv"
"strings"
@@ -43,6 +44,9 @@ var tracer = otel.Tracer("github.com/fleetdm/fleet/v4/server/datastore/mysql")
// This is a variable so it can be adjusted during unit testing.
var countHostSoftwareBatchSize = uint64(100000)
// trailingNonWordChars matches trailing anything not a letter, digit, or underscore
var trailingNonWordChars = regexp.MustCompile(`\W+$`)
// Since a host may have a lot of software items, we need to batch the inserts.
// The maximum number of software items we can insert at one time is governed by max_allowed_packet, which already be set to a high value for MDM bootstrap packages,
// and by the maximum number of placeholders in a prepared statement, which is 65,536. These are already fairly large limits.
@@ -404,7 +408,7 @@ func (ds *Datastore) applyChangesForNewSoftwareDB(
return r, nil
}
existingSoftware, incomingByChecksum, existingTitlesForNewSoftware, existingBundleIDsToUpdate, err := ds.getExistingSoftware(ctx, current, incoming)
existingSoftware, incomingByChecksum, existingTitlesForNewSoftware, err := ds.getExistingSoftware(ctx, current, incoming)
if err != nil {
return r, err
}
@@ -438,54 +442,11 @@ func (ds *Datastore) applyChangesForNewSoftwareDB(
}
r.Inserted = inserted
// Also link existing software that matches by bundle ID
// This handles the case where software has the same bundle ID but different name.
// Since this is a rare case, it is not optimized for performance.
if len(existingBundleIDsToUpdate) > 0 {
bundleInserted, err := ds.linkExistingBundleIDSoftware(ctx, tx, hostID, existingBundleIDsToUpdate)
if err != nil {
return err
}
r.Inserted = append(r.Inserted, bundleInserted...)
// Build map of software IDs to their new names for renaming.
// softwareRenames should match existingBundleIDsToUpdate, but we create this extra map to handle
// software entries that share the same bundle ID as well as other potential corner cases.
softwareRenames := make(map[uint]string, len(existingBundleIDsToUpdate))
// Check inserted software for renames
for _, sw := range r.Inserted {
if sw.BundleIdentifier != "" {
if updSoftwareList, needsUpdate := existingBundleIDsToUpdate[sw.BundleIdentifier]; needsUpdate {
// Use the first software in the list for the name (they should all have the same name)
if len(updSoftwareList) > 0 {
softwareRenames[sw.ID] = updSoftwareList[0].Name
}
}
}
}
// Check existing software for renames
for _, s := range existingSoftware {
if s.BundleIdentifier != nil && *s.BundleIdentifier != "" {
if updSoftwareList, ok := existingBundleIDsToUpdate[*s.BundleIdentifier]; ok {
// Use the first software in the list for the name (they should all have the same name)
if len(updSoftwareList) > 0 {
softwareRenames[s.ID] = updSoftwareList[0].Name
}
}
}
}
if err = updateTargetedBundleIDs(ctx, tx, softwareRenames); err != nil {
return err
}
}
// Use r.Inserted which contains all inserted items (including bundle ID matches)
if err = checkForDeletedInstalledSoftware(ctx, tx, deleted, r.Inserted, hostID); err != nil {
return err
}
if err = updateModifiedHostSoftwareDB(ctx, tx, hostID, current, incoming, existingBundleIDsToUpdate, ds.minLastOpenedAtDiff, ds.logger); err != nil {
if err = updateModifiedHostSoftwareDB(ctx, tx, hostID, current, incoming, ds.minLastOpenedAtDiff, ds.logger); err != nil {
return err
}
@@ -501,108 +462,6 @@ func (ds *Datastore) applyChangesForNewSoftwareDB(
return r, err
}
// updateTargetedBundleIDs updates software names when bundle IDs match but names differ.
// softwareRenames maps software IDs to their new names.
func updateTargetedBundleIDs(ctx context.Context, tx sqlx.ExtContext, softwareRenames map[uint]string) error {
if len(softwareRenames) == 0 {
return nil
}
// Extract software IDs for batch processing
softwareIDs := make([]uint, 0, len(softwareRenames))
for id := range softwareRenames {
softwareIDs = append(softwareIDs, id)
}
const batchSize = 100
err := common_mysql.BatchProcessSimple(softwareIDs, batchSize, func(batch []uint) error {
placeholders := make([]string, len(batch))
args := make([]any, len(batch))
for i, id := range batch {
placeholders[i] = "?"
args[i] = id
}
// During high concurrency situations, we may have multiple transactions attempting to update the same software rows.
// For example, this can happen when many hosts are trying to rename the same software items.
// To avoid long locks or even deadlocks, use UPDATE SKIP LOCKED to skip rows that are already locked by another transaction.
// This means that some software rows may not be updated in this transaction,
// however, eventually they should be updated. This is trading off immediate consistency
// for less lock contention.
lockQuery := fmt.Sprintf(
"SELECT id, name FROM software WHERE id IN (%s) ORDER BY id FOR UPDATE SKIP LOCKED",
strings.Join(placeholders, ","),
)
rows, err := tx.QueryContext(ctx, lockQuery, args...)
if err != nil {
return ctxerr.Wrap(ctx, err, "lock software rows for rename")
}
defer rows.Close()
type lockedRow struct {
id uint
currentName string
}
var lockedRows []lockedRow
for rows.Next() {
var lr lockedRow
if err := rows.Scan(&lr.id, &lr.currentName); err != nil {
return ctxerr.Wrap(ctx, err, "scan locked row")
}
lockedRows = append(lockedRows, lr)
}
if err := rows.Err(); err != nil {
return ctxerr.Wrap(ctx, err, "iterate locked rows")
}
if len(lockedRows) == 0 {
return nil
}
var rowsToUpdate []lockedRow
for _, lr := range lockedRows {
newName := softwareRenames[lr.id]
if lr.currentName != newName {
rowsToUpdate = append(rowsToUpdate, lr)
}
}
if len(rowsToUpdate) == 0 {
return nil
}
updateCases := make([]string, 0, len(rowsToUpdate))
updateCaseArgs := make([]any, 0, len(rowsToUpdate)*2)
updateWhereArgs := make([]any, 0, len(rowsToUpdate))
updateIDs := make([]string, 0, len(rowsToUpdate))
for _, lr := range rowsToUpdate {
newName := softwareRenames[lr.id]
updateCases = append(updateCases, "WHEN ? THEN ?")
updateCaseArgs = append(updateCaseArgs, lr.id, newName)
updateWhereArgs = append(updateWhereArgs, lr.id)
updateIDs = append(updateIDs, "?")
}
updateStmt := fmt.Sprintf(
`UPDATE software SET name = CASE id %s END, name_source = 'bundle_4.67' WHERE id IN (%s)`,
strings.Join(updateCases, " "),
strings.Join(updateIDs, ","),
)
_, err = tx.ExecContext(ctx, updateStmt, append(updateCaseArgs, updateWhereArgs...)...)
if err != nil {
return ctxerr.Wrap(ctx, err, "batch update software names")
}
return nil
})
return err
}
func checkForDeletedInstalledSoftware(ctx context.Context, tx sqlx.ExtContext, deleted []fleet.Software, inserted []fleet.Software,
hostID uint,
) error {
@@ -677,27 +536,20 @@ func (ds *Datastore) getExistingSoftware(
currentSoftware []softwareIDChecksum,
incomingChecksumToSoftware map[string]fleet.Software,
incomingChecksumToTitle map[string]fleet.SoftwareTitle,
existingBundleIDsToUpdate map[string][]fleet.Software,
err error,
) {
// Compute checksums for all incoming software, which we will use for faster retrieval, since checksum is a unique index
incomingChecksumToSoftware = make(map[string]fleet.Software, len(current))
newSoftware := make(map[string]struct{})
incomingBundleIDsToNewSoftwareNames := make(map[string]string)
existingBundleIDsToUpdate = make(map[string][]fleet.Software)
for uniqueName, s := range incoming {
_, ok := current[uniqueName]
if !ok {
checksum, err := s.ComputeRawChecksum()
if err != nil {
return nil, nil, nil, nil, err
return nil, nil, nil, err
}
incomingChecksumToSoftware[string(checksum)] = s
newSoftware[string(checksum)] = struct{}{}
if s.BundleIdentifier != "" {
incomingBundleIDsToNewSoftwareNames[s.BundleIdentifier] = s.Name
}
}
}
@@ -710,45 +562,32 @@ func (ds *Datastore) getExistingSoftware(
// It is OK if the software is not found in the replica DB, because we will then attempt to insert it in the master DB.
currentSoftware, err = getSoftwareIDsByChecksums(ctx, ds.reader(ctx), keys)
if err != nil {
return nil, nil, nil, nil, err
return nil, nil, nil, err
}
for _, currentSoftwareItem := range currentSoftware {
incomingSoftwareItem, ok := incomingChecksumToSoftware[currentSoftwareItem.Checksum]
_, ok := incomingChecksumToSoftware[currentSoftwareItem.Checksum]
if !ok {
// This should never happen. If it does, we have a bug.
return nil, nil, nil, nil, ctxerr.New(
return nil, nil, nil, ctxerr.New(
ctx, fmt.Sprintf("current software: software not found for checksum %s", hex.EncodeToString([]byte(currentSoftwareItem.Checksum))),
)
}
if currentSoftwareItem.BundleIdentifier != nil && currentSoftwareItem.Source == "apps" {
if name, ok := incomingBundleIDsToNewSoftwareNames[*currentSoftwareItem.BundleIdentifier]; ok && name != currentSoftwareItem.Name {
// Then this is a software whose name has changed, so we should update the name
// Copy the incoming software but with the existing software's ID
swWithID := incomingSoftwareItem
swWithID.ID = currentSoftwareItem.ID
existingBundleIDsToUpdate[*currentSoftwareItem.BundleIdentifier] = append(existingBundleIDsToUpdate[*currentSoftwareItem.BundleIdentifier], swWithID)
// Delete this checksum to prevent it from being treated as new software
delete(incomingChecksumToSoftware, currentSoftwareItem.Checksum)
continue
}
}
delete(newSoftware, currentSoftwareItem.Checksum)
}
}
if len(newSoftware) == 0 {
return currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, existingBundleIDsToUpdate, nil
return currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, nil
}
// There's new software, so we try to get the titles already stored in `software_titles` for them.
incomingChecksumToTitle, _, err = ds.getIncomingSoftwareChecksumsToExistingTitles(ctx, newSoftware, incomingChecksumToSoftware)
if err != nil {
return nil, nil, nil, nil, ctxerr.Wrap(ctx, err, "get incoming software checksums to existing titles")
return nil, nil, nil, ctxerr.Wrap(ctx, err, "get incoming software checksums to existing titles")
}
return currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, existingBundleIDsToUpdate, nil
return currentSoftware, incomingChecksumToSoftware, incomingChecksumToTitle, nil
}
// getIncomingSoftwareChecksumsToExistingTitles loads the existing titles for the new incoming software.
@@ -913,6 +752,33 @@ func deleteUninstalledHostSoftwareDB(
return deletedSoftware, nil
}
// longestCommonPrefix finds the longest common prefix among a slice of strings.
// Returns empty string if there's no common prefix.
func longestCommonPrefix(strs []string) string {
if len(strs) == 0 {
return ""
}
if len(strs) == 1 {
return strs[0]
}
firstLen := len(strs[0])
i := 0
for {
if i >= firstLen {
return strs[0]
}
c := strs[0][i]
for _, s := range strs[1:] {
if i >= len(s) || s[i] != c {
return strs[0][:i]
}
}
i++
}
}
// preInsertSoftwareInventory pre-inserts software and software_titles outside the main transaction
// to reduce lock contention. These operations are idempotent due to INSERT IGNORE.
func (ds *Datastore) preInsertSoftwareInventory(
@@ -921,15 +787,37 @@ func (ds *Datastore) preInsertSoftwareInventory(
softwareChecksums map[string]fleet.Software,
existingTitlesForNewSoftware map[string]fleet.SoftwareTitle,
) error {
type titleKey struct {
name string
source string
extensionFor string
bundleID string
isKernel bool
}
// Collect all software that needs to be inserted
needsInsert := make(map[string]fleet.Software)
bundleGroups := make(map[titleKey][]string)
keys := make([]string, 0, len(softwareChecksums))
existingSet := make(map[string]struct{}, len(existingSoftware))
for _, es := range existingSoftware {
existingSet[es.Checksum] = struct{}{}
}
for checksum, sw := range softwareChecksums {
if _, ok := existingSet[checksum]; !ok {
needsInsert[checksum] = sw
keys = append(keys, checksum)
if sw.BundleIdentifier != "" {
key := titleKey{
bundleID: sw.BundleIdentifier,
source: sw.Source,
extensionFor: sw.ExtensionFor,
}
bundleGroups[key] = append(bundleGroups[key], sw.Name)
}
}
}
@@ -937,12 +825,31 @@ func (ds *Datastore) preInsertSoftwareInventory(
return nil
}
// Process in smaller batches to reduce lock time
keys := make([]string, 0, len(needsInsert))
for checksum := range needsInsert {
keys = append(keys, checksum)
bestTitleNames := make(map[titleKey]string)
for key, names := range bundleGroups {
if len(names) > 1 {
// Pick the best represenative name for the group of names
commonPrefix := longestCommonPrefix(names)
commonPrefix = trailingNonWordChars.ReplaceAllString(commonPrefix, "")
if len(commonPrefix) > 0 {
bestTitleNames[key] = commonPrefix
} else {
// Fall back to shortest name
shortest := names[0]
for _, name := range names[1:] {
if len(name) < len(shortest) {
shortest = name
}
}
bestTitleNames[key] = shortest
}
} else if len(names) == 1 {
// Single title or no bundle_identifier
bestTitleNames[key] = names[0]
}
}
// Process in smaller batches to reduce lock time
err := common_mysql.BatchProcessSimple(keys, softwareInventoryInsertBatchSize, func(batchKeys []string) error {
batchSoftware := make(map[string]fleet.Software, len(batchKeys))
for _, key := range batchKeys {
@@ -955,8 +862,20 @@ func (ds *Datastore) preInsertSoftwareInventory(
newTitlesNeeded := make(map[string]fleet.SoftwareTitle)
for checksum, sw := range batchSoftware {
if _, ok := existingTitlesForNewSoftware[checksum]; !ok {
titleName := sw.Name
if sw.BundleIdentifier != "" {
key := titleKey{
bundleID: sw.BundleIdentifier,
source: sw.Source,
extensionFor: sw.ExtensionFor,
}
if computedName, exists := bestTitleNames[key]; exists {
titleName = computedName
}
}
st := fleet.SoftwareTitle{
Name: sw.Name,
Name: titleName,
Source: sw.Source,
ExtensionFor: sw.ExtensionFor,
IsKernel: sw.IsKernel,
@@ -980,15 +899,7 @@ func (ds *Datastore) preInsertSoftwareInventory(
}
if len(newTitlesNeeded) > 0 {
// Deduplicate titles before insertion to avoid unnecessary duplicate INSERTs
type titleKey struct {
name string
source string
extensionFor string
bundleID string
isKernel bool
}
uniqueTitlesToInsert := make(map[titleKey]fleet.SoftwareTitle, len(newTitlesNeeded))
uniqueTitlesToInsert := make(map[titleKey]fleet.SoftwareTitle)
for _, title := range newTitlesNeeded {
bundleID := ""
if title.BundleIdentifier != nil {
@@ -1001,11 +912,14 @@ func (ds *Datastore) preInsertSoftwareInventory(
bundleID: bundleID,
isKernel: title.IsKernel,
}
uniqueTitlesToInsert[key] = title
if _, exists := uniqueTitlesToInsert[key]; !exists {
uniqueTitlesToInsert[key] = title
}
}
// Insert software titles
const numberOfArgsPerSoftwareTitles = 5
const numberOfArgsPerSoftwareTitles = 6
titlesValues := strings.TrimSuffix(strings.Repeat("(?,?,?,?,?,?),", len(uniqueTitlesToInsert)), ",")
titlesStmt := fmt.Sprintf("INSERT IGNORE INTO software_titles (name, source, extension_for, bundle_identifier, is_kernel, application_id) VALUES %s", titlesValues)
titlesArgs := make([]any, 0, len(uniqueTitlesToInsert)*numberOfArgsPerSoftwareTitles)
@@ -1027,20 +941,25 @@ func (ds *Datastore) preInsertSoftwareInventory(
BundleIdentifier *string `db:"bundle_identifier"`
}
// Build query to retrieve title IDs using the same unique titles we inserted
titlePlaceholders := strings.TrimSuffix(strings.Repeat("(?,?,?,?),", len(uniqueTitlesToInsert)), ",")
queryArgs := make([]interface{}, 0, len(uniqueTitlesToInsert)*4)
for tk := range uniqueTitlesToInsert {
title := uniqueTitlesToInsert[tk]
bundleID := ""
if uniqueTitlesToInsert[tk].BundleIdentifier != nil {
bundleID = *uniqueTitlesToInsert[tk].BundleIdentifier
if title.BundleIdentifier != nil {
bundleID = *title.BundleIdentifier
}
queryArgs = append(queryArgs, tk.name, tk.source, tk.extensionFor, bundleID)
firstArg := title.Name
if bundleID != "" {
firstArg = bundleID
}
queryArgs = append(queryArgs, firstArg, title.Source, title.ExtensionFor, bundleID)
}
queryTitles := fmt.Sprintf(`SELECT id, name, source, extension_for, bundle_identifier
FROM software_titles
WHERE (name, source, extension_for, COALESCE(bundle_identifier, '')) IN (%s)`, titlePlaceholders)
WHERE (COALESCE(bundle_identifier, name), source, extension_for, COALESCE(bundle_identifier, '')) IN (%s)`, titlePlaceholders)
if err := sqlx.SelectContext(ctx, tx, &titlesData, queryTitles, queryArgs...); err != nil {
return ctxerr.Wrap(ctx, err, "select software titles")
@@ -1057,7 +976,14 @@ func (ds *Datastore) preInsertSoftwareInventory(
if title.BundleIdentifier != nil {
titleBundleID = *title.BundleIdentifier
}
if td.Name == title.Name && td.Source == title.Source && td.ExtensionFor == title.ExtensionFor && bundleID == titleBundleID {
// For apps with bundle_identifier, match by bundle_identifier (since we may have picked a different name)
// For others, match by name
nameMatches := td.Name == title.Name
if bundleID != "" && titleBundleID != "" {
// Both have bundle_identifier - match by bundle_identifier instead of name
nameMatches = true
}
if nameMatches && td.Source == title.Source && td.ExtensionFor == title.ExtensionFor && bundleID == titleBundleID {
titleIDsByChecksum[checksum] = td.ID
// Don't break here - multiple checksums can map to the same title
// (e.g., when software has same truncated name but different versions (very rare))
@@ -1067,7 +993,7 @@ func (ds *Datastore) preInsertSoftwareInventory(
}
// Insert software entries
const numberOfArgsPerSoftware = 11
const numberOfArgsPerSoftware = 12
values := strings.TrimSuffix(
strings.Repeat("(?,?,?,?,?,?,?,?,?,?,?,?),", len(batchKeys)), ",",
)
@@ -1133,81 +1059,6 @@ func (ds *Datastore) preInsertSoftwareInventory(
return err
}
// linkExistingBundleIDSoftware links existing software entries that match by bundle ID to the host.
// This handles the case where incoming software has the same bundle ID as existing software but a different name.
func (ds *Datastore) linkExistingBundleIDSoftware(
ctx context.Context,
tx sqlx.ExtContext,
hostID uint,
existingBundleIDsToUpdate map[string][]fleet.Software,
) ([]fleet.Software, error) {
if len(existingBundleIDsToUpdate) == 0 {
return nil, nil
}
// Collect all software IDs to verify they still exist
softwareIDs := make([]uint, 0, len(existingBundleIDsToUpdate))
for _, softwareList := range existingBundleIDsToUpdate {
for _, software := range softwareList {
// The software.ID should already be set from getExistingSoftware
if software.ID == 0 {
return nil, ctxerr.New(ctx, "software ID not set for bundle ID match")
}
softwareIDs = append(softwareIDs, software.ID)
}
}
// Verify software still exists (just like we do in linkSoftwareToHost)
// This prevents creating orphaned references if software was deleted between pre-insertion and now
// This DB call could be removed to squeeze our a little more performance at the risk of orphaned references.
stmt, args, err := sqlx.In(`SELECT id FROM software WHERE id IN (?)`, softwareIDs)
if err != nil {
return nil, ctxerr.Wrap(ctx, err, "build query for existing software verification")
}
var existingIDs []uint
if err := sqlx.SelectContext(ctx, tx, &existingIDs, stmt, args...); err != nil {
return nil, ctxerr.Wrap(ctx, err, "verify existing bundle ID software")
}
// Build a set of existing IDs for quick lookup
existingIDSet := make(map[uint]struct{}, len(existingIDs))
for _, id := range existingIDs {
existingIDSet[id] = struct{}{}
}
var insertsHostSoftware []any
var insertedSoftware []fleet.Software
for _, softwareList := range existingBundleIDsToUpdate {
for _, software := range softwareList {
// Only link if software still exists
if _, ok := existingIDSet[software.ID]; ok {
insertsHostSoftware = append(insertsHostSoftware, hostID, software.ID, software.LastOpenedAt)
insertedSoftware = append(insertedSoftware, software)
} else {
// Log missing software but continue
level.Warn(ds.logger).Log(
"msg", "bundle ID software not found after pre-insertion",
"software_id", software.ID,
"name", software.Name,
"bundle_id", software.BundleIdentifier,
)
}
}
}
if len(insertsHostSoftware) > 0 {
values := strings.TrimSuffix(strings.Repeat("(?,?,?),", len(insertsHostSoftware)/3), ",")
stmt := fmt.Sprintf(`INSERT IGNORE INTO host_software (host_id, software_id, last_opened_at) VALUES %s`, values)
if _, err := tx.ExecContext(ctx, stmt, insertsHostSoftware...); err != nil {
return nil, ctxerr.Wrap(ctx, err, "link existing bundle ID software")
}
}
return insertedSoftware, nil
}
// linkSoftwareToHost links pre-inserted software to a host.
// This assumes software inventory entries already exist.
func (ds *Datastore) linkSoftwareToHost(
@@ -1297,7 +1148,6 @@ func updateModifiedHostSoftwareDB(
hostID uint,
currentMap map[string]fleet.Software,
incomingMap map[string]fleet.Software,
existingBundleIDsToUpdate map[string][]fleet.Software,
minLastOpenedAtDiff time.Duration,
logger log.Logger,
) error {
@@ -1308,21 +1158,16 @@ func updateModifiedHostSoftwareDB(
if !ok {
continue
}
// if the new software has no last opened timestamp, we only
// update if the current software has no last opened timestamp
// and is marked as having a name change.
// if the new software has no last opened timestamp, log if the current one did
// (but only for non-apps sources, as apps sources are managed by osquery)
if newSw.LastOpenedAt == nil {
if _, ok := existingBundleIDsToUpdate[newSw.BundleIdentifier]; ok && curSw.LastOpenedAt == nil {
keysToUpdate = append(keysToUpdate, key)
}
// Log cases where the new software has no last opened timestamp, the current software does,
// and the software is marked as having a name change.
// This is expected on macOS, but not on windows/linux.
if ok && curSw.LastOpenedAt != nil && newSw.Source != "apps" {
level.Warn(logger).Log(
"msg", "updateModifiedHostSoftwareDB: last opened at is nil for new software, but not for current software",
"new_software", newSw.Name, "current_software", curSw.Name,
"bundle_identifier", newSw.BundleIdentifier,
if curSw.LastOpenedAt != nil && newSw.Source != "apps" {
level.Info(logger).Log(
"msg", "software last_opened_at changed to nil",
"host_id", hostID,
"software_id", curSw.ID,
"software_name", newSw.Name,
"source", newSw.Source,
)
}
continue
+177 -88
View File
@@ -99,6 +99,7 @@ func TestSoftware(t *testing.T) {
{"InventoryPendingSoftware", testInventoryPendingSoftware},
{"PreInsertSoftwareInventory", testPreInsertSoftwareInventory},
{"ListHostSoftwareWithExtensionFor", testListHostSoftwareWithExtensionFor},
{"LongestCommonPrefix", testLongestCommonPrefix},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
@@ -1884,10 +1885,6 @@ func testUpdateHostSoftware(t *testing.T, ds *Datastore) {
// When software with the same bundle ID but different name is added, the system
// reuses the existing software entry (matched by bundle ID) and links it to the host
func testUpdateHostSoftwareSameBundleIDDifferentNames(t *testing.T, ds *Datastore) {
// TEMPORARILY SKIPPED: updateTargetedBundleIDs is commented out for performance reasons
// TODO: Re-enable when updateTargetedBundleIDs is re-enabled
t.Skip("Skipping test: updateTargetedBundleIDs is temporarily disabled for performance reasons")
ctx := t.Context()
host := test.NewHost(t, ds, "bundle-host", "", "bundlekey", "bundleuuid", time.Now())
@@ -1903,47 +1900,98 @@ func testUpdateHostSoftwareSameBundleIDDifferentNames(t *testing.T, ds *Datastor
require.NoError(t, err)
require.Len(t, host.Software, 1)
require.Equal(t, "GoLand.app", host.Software[0].Name)
originalSoftwareID := host.Software[0].ID
// Now update with the same bundle ID but different name
// The behavior depends on how the system handles bundle ID matching
// Despite having the same bundle id, the software is added with the new name
sw = []fleet.Software{
{Name: "GoLand 2.app", Version: "2024.3", Source: "apps", BundleIdentifier: "com.jetbrains.goland"},
}
_, err = ds.UpdateHostSoftware(ctx, host.ID, sw)
require.NoError(t, err)
// The getExistingSoftware function matches by bundle ID when present,
// so it links to the existing software entry and updates the name
err = ds.LoadHostSoftware(ctx, host, false)
require.NoError(t, err)
require.Len(t, host.Software, 1)
// The existing software entry is reused (matched by bundle ID) and name is updated
require.Equal(t, "GoLand 2.app", host.Software[0].Name, "Name should be updated to reflect what's on the host")
require.Equal(t, originalSoftwareID, host.Software[0].ID, "Should reuse the same software row")
// Verify the name_source was updated
var nameSource string
err = ds.writer(ctx).GetContext(ctx, &nameSource,
`SELECT name_source FROM software WHERE id = ?`, originalSoftwareID)
require.NoError(t, err)
require.Equal(t, "bundle_4.67", nameSource, "Name source should indicate bundle ID match")
// Verify only one software title exists
// Verify only one software title exists (both software entries map to same title by bundle_identifier)
var titleCount int
err = ds.writer(ctx).GetContext(ctx, &titleCount,
`SELECT COUNT(DISTINCT id) FROM software_titles WHERE bundle_identifier = ?`,
"com.jetbrains.goland")
require.NoError(t, err)
require.Equal(t, 1, titleCount, "Should have only one software title for the bundle ID")
require.Equal(t, 1, titleCount)
// Verify only one software entry exists with this bundle ID
var softwareCount int
err = ds.writer(ctx).GetContext(ctx, &softwareCount,
`SELECT COUNT(DISTINCT id) FROM software WHERE bundle_identifier = ?`,
// Verify two software entries exist with this bundle ID (different names, same bundle_identifier)
var softwareNames []string
err = ds.writer(ctx).SelectContext(ctx, &softwareNames,
`SELECT name FROM software WHERE bundle_identifier = ? ORDER BY name`,
"com.jetbrains.goland")
require.NoError(t, err)
require.Equal(t, 1, softwareCount, "Should have only one software entry for the bundle ID")
require.Len(t, softwareNames, 2)
require.Equal(t, []string{"GoLand 2.app", "GoLand.app"}, softwareNames)
// Helper app edge case:
// We have a main app with a name and bundle id
// We have two helper apps with the same bundle id but different name
sw = []fleet.Software{
{Name: "Postman", Version: "11.60.2", Source: "apps", BundleIdentifier: "com.postmanlabs.mac"},
{Name: "Postman Helper (GPU)", Version: "", Source: "apps", BundleIdentifier: "com.postmanlabs.mac.helper"},
{Name: "Postman Helper (Renderer)", Version: "", Source: "apps", BundleIdentifier: "com.postmanlabs.mac.helper"},
}
_, err = ds.UpdateHostSoftware(ctx, host.ID, sw)
require.NoError(t, err)
err = ds.LoadHostSoftware(ctx, host, false)
require.NoError(t, err)
require.Len(t, host.Software, 3)
var softwareRecords []struct {
Name string `db:"name"`
BundleIdentifier string `db:"bundle_identifier"`
}
err = ds.writer(ctx).SelectContext(ctx, &softwareRecords,
`SELECT name, bundle_identifier FROM software WHERE bundle_identifier = ? OR bundle_identifier = ?`,
"com.postmanlabs.mac", "com.postmanlabs.mac.helper")
require.NoError(t, err)
require.Len(t, softwareRecords, 3)
for _, softwareRecord := range softwareRecords {
switch softwareRecord.Name {
case "Postman":
require.Equal(t, "com.postmanlabs.mac", softwareRecord.BundleIdentifier)
case "Postman Helper (GPU)", "Postman Helper (Renderer)":
require.Equal(t, "com.postmanlabs.mac.helper", softwareRecord.BundleIdentifier)
default:
t.Fatalf("Unexpected software name: %s", softwareRecord.Name)
}
}
// Re-ingest helper apps with new names
sw = []fleet.Software{
{Name: "Postman 2", Version: "11.60.2", Source: "apps", BundleIdentifier: "com.postmanlabs.mac"},
{Name: "Postman Helper 2 (GPU)", Version: "", Source: "apps", BundleIdentifier: "com.postmanlabs.mac.helper"},
{Name: "Postman Helper 2 (Renderer)", Version: "", Source: "apps", BundleIdentifier: "com.postmanlabs.mac.helper"},
}
_, err = ds.UpdateHostSoftware(ctx, host.ID, sw)
require.NoError(t, err)
err = ds.writer(ctx).SelectContext(ctx, &softwareRecords,
`SELECT name, bundle_identifier FROM software WHERE bundle_identifier = ? OR bundle_identifier = ?`,
"com.postmanlabs.mac", "com.postmanlabs.mac.helper")
require.NoError(t, err)
require.Len(t, softwareRecords, 6)
for _, softwareRecord := range softwareRecords {
switch softwareRecord.Name {
case "Postman", "Postman 2":
require.Equal(t, "com.postmanlabs.mac", softwareRecord.BundleIdentifier)
case "Postman Helper (GPU)", "Postman Helper (Renderer)", "Postman Helper 2 (GPU)", "Postman Helper 2 (Renderer)":
require.Equal(t, "com.postmanlabs.mac.helper", softwareRecord.BundleIdentifier)
default:
t.Fatalf("Unexpected software name: %s", softwareRecord.Name)
}
}
}
// Test edge case: Software with same name but different bundle identifiers
@@ -1982,12 +2030,9 @@ func testUpdateHostSoftwareSameNameDifferentBundleIDs(t *testing.T, ds *Datastor
}
// Test edge case: Multiple software entries with the same bundle ID
// This validates that bundle ID renaming affects all software with that bundle ID
// This validates that when software with the same bundle ID but different names
// are added from different hosts, we add software entries for each name
func testUpdateHostSoftwareMultipleSameBundleID(t *testing.T, ds *Datastore) {
// TEMPORARILY SKIPPED: updateTargetedBundleIDs is commented out for performance reasons
// TODO: Re-enable when updateTargetedBundleIDs is re-enabled
t.Skip("Skipping test: updateTargetedBundleIDs is temporarily disabled for performance reasons")
ctx := t.Context()
host1 := test.NewHost(t, ds, "multi-bundle-host1", "", "multikey1", "multiuuid1", time.Now())
host2 := test.NewHost(t, ds, "multi-bundle-host2", "", "multikey2", "multiuuid2", time.Now())
@@ -2015,7 +2060,7 @@ func testUpdateHostSoftwareMultipleSameBundleID(t *testing.T, ds *Datastore) {
require.NoError(t, err)
// Step 3: Host3 reports the SAME software but with a different name
// This should trigger renaming of ALL software with that bundle ID
// This should not rename all software, it should create a new software entry
sw3 := []fleet.Software{
{Name: "GoLand 2024.app", Version: "2024.2", Source: "apps", BundleIdentifier: "com.jetbrains.goland"},
{Name: "GoLand 2024.app", Version: "2024.3-beta", Source: "apps", BundleIdentifier: "com.jetbrains.goland"},
@@ -2023,7 +2068,7 @@ func testUpdateHostSoftwareMultipleSameBundleID(t *testing.T, ds *Datastore) {
_, err = ds.UpdateHostSoftware(ctx, host3.ID, sw3)
require.NoError(t, err)
// Step 4: Verify the renaming behavior
// Step 4: Verify insertion into software behavior
var updatedSoftware []struct {
ID uint `db:"id"`
Name string `db:"name"`
@@ -2035,39 +2080,62 @@ func testUpdateHostSoftwareMultipleSameBundleID(t *testing.T, ds *Datastore) {
"com.jetbrains.goland")
require.NoError(t, err)
// Should have exactly 2 software entries (one per version)
require.Len(t, updatedSoftware, 2, "Should have exactly 2 software entries (one per version)")
// Should have exactly 4 software entries
require.Len(t, updatedSoftware, 4, "Should have exactly 4 software entries")
// Verify we have both versions for each name
golandAppVersions := make(map[string]bool)
goland2024AppVersions := make(map[string]bool)
// Both entries should be renamed to "GoLand 2024.app"
for _, sw := range updatedSoftware {
t.Logf("Software: ID=%d, Name=%s, Version=%s, NameSource=%s", sw.ID, sw.Name, sw.Version, sw.NameSource)
require.Equal(t, "GoLand 2024.app", sw.Name, "All software with same bundle ID should be renamed")
require.Equal(t, "bundle_4.67", sw.NameSource, "Renamed software should have bundle_4.67 source")
switch sw.Name {
case "GoLand.app":
golandAppVersions[sw.Version] = true
case "GoLand 2024.app":
goland2024AppVersions[sw.Version] = true
default:
t.Fatalf("Unexpected software name: %s", sw.Name)
}
}
// Verify that host1 and host2 now see the renamed software
require.Len(t, golandAppVersions, 2, "Should have 2 versions of GoLand.app")
require.True(t, golandAppVersions["2024.2"], "Should have GoLand.app v2024.2")
require.True(t, golandAppVersions["2024.3-beta"], "Should have GoLand.app v2024.3-beta")
require.Len(t, goland2024AppVersions, 2, "Should have 2 versions of GoLand 2024.app")
require.True(t, goland2024AppVersions["2024.2"], "Should have GoLand 2024.app v2024.2")
require.True(t, goland2024AppVersions["2024.3-beta"], "Should have GoLand 2024.app v2024.3-beta")
// Verify that each host sees only their software (no renaming happens)
err = ds.LoadHostSoftware(ctx, host1, false)
require.NoError(t, err)
require.Len(t, host1.Software, 2, "Host1 should still have 2 software entries")
require.Len(t, host1.Software, 2, "Host1 should have 2 software entries")
for _, s := range host1.Software {
require.Equal(t, "GoLand 2024.app", s.Name, "Host1 should see renamed software")
require.Equal(t, "GoLand.app", s.Name, "Host1 software should be GoLand.app")
}
err = ds.LoadHostSoftware(ctx, host2, false)
require.NoError(t, err)
require.Len(t, host2.Software, 1, "Host2 should have 1 software entry")
require.Equal(t, "GoLand 2024.app", host2.Software[0].Name, "Host2 should see renamed software")
require.Equal(t, "GoLand.app", host2.Software[0].Name, "Host2 software should be GoLand.app")
err = ds.LoadHostSoftware(ctx, host3, false)
require.NoError(t, err)
require.Len(t, host3.Software, 2, "Host3 should have 2 software entries")
for _, s := range host3.Software {
require.Equal(t, "GoLand 2024.app", s.Name, "Host3 software should be GoLand 2024.app")
}
}
// Test for the bug where multiple software with the same bundle ID causes
// "software not found for checksum" errors during bundle ID rename operations.
// "software not found for checksum" errors
// This test specifically validates that ALL software entries with the same
// bundle ID are properly linked to hosts when renaming occurs.
// bundle ID are properly linked to hosts
func testUpdateHostSoftwareMultipleChecksumsPerBundleID(t *testing.T, ds *Datastore) {
ctx := t.Context()
// Note: Basic multiple versions scenario is already covered in testUpdateHostSoftwareMultipleSameBundleID
// This test focuses on the specific bug fix for renamed apps with many versions
// This test focuses on the specific bug fix for apps with many versions
// First, establish the software with host1 - using 10 versions to stress test
host1 := test.NewHost(t, ds, "rename-test-host1", "", "rename-key1", "rename-uuid1", time.Now())
@@ -2091,8 +2159,7 @@ func testUpdateHostSoftwareMultipleChecksumsPerBundleID(t *testing.T, ds *Datast
require.NoError(t, err)
require.Len(t, host1.Software, 10, "Host1 should have all 10 versions")
// Host2 reports the same software but renamed (user renamed the apps)
// This triggers the bundle ID rename logic and tests the bug fix
// Host2 reports the same software but different names
host2 := test.NewHost(t, ds, "rename-test-host2", "", "rename-key2", "rename-uuid2", time.Now())
var renamedSoftware []fleet.Software
@@ -2110,7 +2177,7 @@ func testUpdateHostSoftwareMultipleChecksumsPerBundleID(t *testing.T, ds *Datast
require.NoError(t, err, "Should handle renamed apps with 10 versions without 'software not found for checksum' error")
assert.NotNil(t, result)
// Verify the rename was processed in the database
// Verify both names exist in the database (no renaming occurs)
var dbSoftware []struct {
Name string `db:"name"`
Version string `db:"version"`
@@ -2118,31 +2185,47 @@ func testUpdateHostSoftwareMultipleChecksumsPerBundleID(t *testing.T, ds *Datast
}
err = ds.writer(ctx).SelectContext(ctx, &dbSoftware,
`SELECT name, version, name_source FROM software
WHERE bundle_identifier = ? ORDER BY version`,
WHERE bundle_identifier = ? ORDER BY name, version`,
"com.stresstest.app")
require.NoError(t, err)
require.Len(t, dbSoftware, 10, "Should have 10 software entries in database")
require.Len(t, dbSoftware, 20, "Should have 20 software entries: 10 for each name")
// All should be renamed
// Verify we have 10 of each name
testAppCount := 0
testAppRenamedCount := 0
for _, sw := range dbSoftware {
assert.Equal(t, "TestApp Renamed.app", sw.Name, "All software should use the new name")
assert.Equal(t, "bundle_4.67", sw.NameSource, "Renamed software should have bundle_4.67 source")
switch sw.Name {
case "TestApp.app":
testAppCount++
case "TestApp Renamed.app":
testAppRenamedCount++
}
}
assert.Equal(t, 10, testAppCount, "Should have 10 'TestApp.app' entries")
assert.Equal(t, 10, testAppRenamedCount, "Should have 10 'TestApp Renamed.app' entries")
// Verify that host1 still has its original software
err = ds.LoadHostSoftware(ctx, host1, false)
require.NoError(t, err)
assert.Len(t, host1.Software, 10, "Host1 should still have all 10 versions")
for _, sw := range host1.Software {
assert.Equal(t, "TestApp.app", sw.Name, "Host1 should see original name")
}
// Most importantly, verify that host2 has ALL 10 versions linked (this was the bug)
// Verify that host2 has ALL 10 versions linked
err = ds.LoadHostSoftware(ctx, host2, false)
require.NoError(t, err)
assert.Len(t, host2.Software, 10, "Host2 should have all 10 versions linked (bug fix verification)")
assert.Len(t, host2.Software, 10, "Host2 should have all 10 versions linked")
// Verify all versions are present
// Verify all versions are present for host2
versions := make(map[string]bool)
for _, sw := range host2.Software {
versions[sw.Version] = true
assert.Equal(t, "TestApp Renamed.app", sw.Name, "Should see renamed app")
assert.Equal(t, "TestApp Renamed.app", sw.Name, "Host2 should see its own name")
}
for i := 0; i < 10; i++ {
version := fmt.Sprintf("1.%d.0", i)
assert.True(t, versions[version], "Should have version %s", version)
assert.True(t, versions[version], "Host2 should have version %s", version)
}
}
@@ -9194,10 +9277,6 @@ func testPreInsertSoftwareInventory(t *testing.T, ds *Datastore) {
// testUpdateHostBundleIDRenameOnlyNoNewSoftware tests if a host reports ONLY renamed software
// (same bundle ID, different name) with NO new software
func testUpdateHostBundleIDRenameOnlyNoNewSoftware(t *testing.T, ds *Datastore) {
// TEMPORARILY SKIPPED: updateTargetedBundleIDs is commented out for performance reasons
// TODO: Re-enable when updateTargetedBundleIDs is re-enabled
t.Skip("Skipping test: updateTargetedBundleIDs is temporarily disabled for performance reasons")
ctx := t.Context()
host := test.NewHost(t, ds, "rename-test-host", "", "renamekey", "renameuuid", time.Now())
@@ -9219,7 +9298,6 @@ func testUpdateHostBundleIDRenameOnlyNoNewSoftware(t *testing.T, ds *Datastore)
}
// Report ONLY renamed software (same bundle IDs, different names)
// This is the edge case: NO new software, ONLY renames
renamedSoftware := []fleet.Software{
{Name: "Renamed.app", Version: "1.0", Source: "apps", BundleIdentifier: "com.example.app"},
{Name: "AlsoRenamed.app", Version: "2.0", Source: "apps", BundleIdentifier: "com.example.another"},
@@ -9229,37 +9307,33 @@ func testUpdateHostBundleIDRenameOnlyNoNewSoftware(t *testing.T, ds *Datastore)
_, err = ds.UpdateHostSoftware(ctx, host.ID, renamedSoftware)
require.NoError(t, err)
// Verify the software entries were reused (not duplicated)
// Verify the host only has 2 pieces of sofware
err = ds.LoadHostSoftware(ctx, host, false)
require.NoError(t, err)
require.Len(t, host.Software, 2, "Should still have exactly 2 software entries")
// Verify the IDs are the same (software was reused, not recreated)
// Verify the IDs are are not the same
for _, s := range host.Software {
originalID, ok := originalIDs[s.BundleIdentifier]
require.True(t, ok, "Bundle ID %s should exist", s.BundleIdentifier)
require.Equal(t, originalID, s.ID,
"Software ID should be reused for bundle ID %s", s.BundleIdentifier)
require.NotEqual(t, originalID, s.ID,
"Software ID should not be reused for bundle ID %s", s.BundleIdentifier)
}
// Verify no duplicate software entries were created
// Verify new software entries were created
var softwareCount int
err = ds.writer(ctx).GetContext(ctx, &softwareCount,
`SELECT COUNT(DISTINCT id) FROM software
WHERE bundle_identifier IN ('com.example.app', 'com.example.another')`)
require.NoError(t, err)
require.Equal(t, 2, softwareCount, "Should have exactly 2 software entries, not duplicates")
require.Equal(t, 4, softwareCount, "Should have exactly 4 software entries")
}
// testUpdateHostBundleIDRenameWithNewSoftware tests the edge case where a host reports BOTH:
// 1. New software that needs to be inserted
// 2. Existing software with renamed bundle IDs that needs updating
// 2. Existing software with renamed bundle IDs
// This tests that both operations work correctly in the same update.
func testUpdateHostBundleIDRenameWithNewSoftware(t *testing.T, ds *Datastore) {
// TEMPORARILY SKIPPED: updateTargetedBundleIDs is commented out for performance reasons
// TODO: Re-enable when updateTargetedBundleIDs is re-enabled
t.Skip("Skipping test: updateTargetedBundleIDs is temporarily disabled for performance reasons")
ctx := t.Context()
host := test.NewHost(t, ds, "mixed-test-host", "", "mixedkey", "mixeduuid", time.Now())
@@ -9276,9 +9350,8 @@ func testUpdateHostBundleIDRenameWithNewSoftware(t *testing.T, ds *Datastore) {
require.Len(t, host.Software, 1)
slackOriginalID := host.Software[0].ID
// Step 2: Report BOTH renamed software AND new software in the same update
mixedUpdate := []fleet.Software{
// Renamed existing software (same bundle ID, different name)
// same bundle ID, different name
{Name: "Slack 2.app", Version: "1.0.0", Source: "apps", BundleIdentifier: "com.tinyspeck.slackmacgap"},
// Brand new software
{Name: "Chrome.app", Version: "110.0", Source: "apps", BundleIdentifier: "com.google.Chrome"},
@@ -9303,17 +9376,8 @@ func testUpdateHostBundleIDRenameWithNewSoftware(t *testing.T, ds *Datastore) {
switch s.BundleIdentifier {
case "com.tinyspeck.slackmacgap":
foundSlack = true
// Verify Slack was renamed and ID was reused
require.Equal(t, "Slack 2.app", s.Name, "Slack should be renamed")
require.Equal(t, slackOriginalID, s.ID, "Slack should reuse the same ID")
// Verify name_source was updated
var nameSource string
err = ds.writer(ctx).GetContext(ctx, &nameSource,
`SELECT name_source FROM software WHERE id = ?`, s.ID)
require.NoError(t, err)
require.Equal(t, "bundle_4.67", nameSource, "Name source should indicate bundle ID match")
require.NotEqual(t, slackOriginalID, s.ID)
case "com.google.Chrome":
foundChrome = true
require.Equal(t, "Chrome.app", s.Name)
@@ -9328,18 +9392,19 @@ func testUpdateHostBundleIDRenameWithNewSoftware(t *testing.T, ds *Datastore) {
}
}
require.True(t, foundSlack, "Should find renamed Slack")
require.True(t, foundSlack, "Should find new Slack")
require.True(t, foundChrome, "Should find new Chrome")
require.True(t, foundCustomTool, "Should find new CustomTool")
// Verify no duplicate software entries were created
// Verify two slack entries exist in the software table
var softwareCount int
err = ds.writer(ctx).GetContext(ctx, &softwareCount,
`SELECT COUNT(DISTINCT id) FROM software WHERE bundle_identifier = 'com.tinyspeck.slackmacgap'`)
require.NoError(t, err)
require.Equal(t, 1, softwareCount, "Should have exactly 1 Slack software entry")
require.Equal(t, 2, softwareCount, "Should have exactly 2 Slack software entries")
// Verify titles were created correctly
// A new one should not have been created for Slack 2.app
var titleCount int
err = ds.writer(ctx).GetContext(ctx, &titleCount,
`SELECT COUNT(DISTINCT id) FROM software_titles`)
@@ -9542,6 +9607,30 @@ func testListHostSoftwareWithExtensionFor(t *testing.T, ds *Datastore) {
require.Equal(t, "", regularApp.ExtensionFor)
}
func testLongestCommonPrefix(t *testing.T, ds *Datastore) {
tests := []struct {
input []string
expected string
}{
{input: []string{}, expected: ""},
{input: []string{"no_common1", "another_one3"}, expected: ""},
{input: []string{"single"}, expected: "single"},
{input: []string{"prefix_common", "prefix_common_suffix1", "prefix_common_suffix2"}, expected: "prefix_common"},
{input: []string{"common_prefix_suffix1", "common_prefix_suffix2", "common_prefix"}, expected: "common_prefix"},
{input: []string{"same", "same", "same"}, expected: "same"},
{input: []string{"partial_common1", "partial_common2", "none"}, expected: ""},
{input: []string{"", "softwarename"}, expected: ""},
{input: []string{"softwarename", "prefix_common", "prefix_common"}, expected: ""},
}
for _, tt := range tests {
t.Run(fmt.Sprintf("%v", tt.input), func(t *testing.T) {
result := longestCommonPrefix(tt.input)
require.Equal(t, tt.expected, result)
})
}
}
// Helper function to find software by name and extension_for
func findSoftware(sw []*fleet.HostSoftwareWithInstaller, name, extensionFor string) *fleet.HostSoftwareWithInstaller {
for _, s := range sw {
+3 -6
View File
@@ -149,15 +149,12 @@ func (s Software) ToUniqueStr() string {
// The calculation must match the one in softwareChecksumComputedColumn
func (s Software) ComputeRawChecksum() ([]byte, error) {
h := md5.New() //nolint:gosec // This hash is used as a DB optimization for software row lookup, not security
cols := []string{s.Version, s.Source, s.BundleIdentifier, s.Release, s.Arch, s.Vendor, s.ExtensionFor, s.ExtensionID}
// Only incorporate name if the Software is not a macOS app, because names on macOS are easily
// mutable and can lead to unintentional duplicates of Software in Fleet.
if s.Source != "apps" {
cols = append([]string{s.Name}, cols...)
}
cols := []string{s.Version, s.Source, s.BundleIdentifier, s.Release, s.Arch, s.Vendor, s.ExtensionFor, s.ExtensionID, s.Name}
if s.ApplicationID != nil && *s.ApplicationID != "" {
cols = append(cols, *s.ApplicationID)
}
_, err := fmt.Fprint(h, strings.Join(cols, "\x00"))
if err != nil {
return nil, err