Discharge7443/fleet
1
0
Fork 0
Code Issues Pull Requests Actions 8 Packages Projects Releases Wiki Activity

Fixed DB lock contention during vulnerability cron's software cleanup that caused failures under load (#41375)

Browse Source 
<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #41374

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.

## Testing

- [x] QA'd all new/changed functionality manually

For unreleased bug fixes in a release candidate, one of:

- [x] Alerted the release DRI if additional load testing is needed

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **Bug Fixes**
* Resolved database lock contention that occurred during software
cleanup operations, which previously caused failures under heavy load.
The cleanup process now uses an optimized batched approach for improved
reliability and performance.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Victor Lyuboslavsky
2026-03-10 13:44:10 -05:00
committed by GitHub
parent 860f5a0ec5
commit 989e503bf5
2 changed files with 42 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
changes/41374-unused-software
+1
View File
@@ -0,0 +1 @@
* Fixed DB lock contention during vulnerability cron's software cleanup that caused failures under load.
server/datastore/mysql/software.go
+41 -16
View File
@@ -63,6 +63,10 @@ var softwareInsertBatchSize = 1000
// outside the main software ingestion transaction. Smaller batches reduce lock contention.
var softwareInventoryInsertBatchSize = 100
// cleanupBatchSize controls how many orphaned software rows are deleted per batch during SyncHostsSoftware cleanup.
// Smaller batches hold locks for shorter durations, reducing contention with concurrent software ingestion.
var cleanupBatchSize = 1000
func softwareSliceToMap(softwareItems []fleet.Software) map[string]fleet.Software {
result := make(map[string]fleet.Software, len(softwareItems))
for _, s := range softwareItems {
@@ -2640,19 +2644,6 @@ func (ds *Datastore) SyncHostsSoftware(ctx context.Context, updatedAt time.Time)
updated_at = VALUES(updated_at)`
valuesPart = `(?, ?, ?, ?, ?),`
// We must ensure that software is not in host_software table before deleting it.
// This prevents a race condition where a host just added the software, but it is not part of software_host_counts yet.
// When a host adds software, software table and host_software table are updated in the same transaction.
cleanupSoftwareStmt = `
DELETE s
FROM software s
LEFT JOIN software_host_counts shc
ON s.id = shc.software_id
WHERE
shc.software_id IS NULL AND
NOT EXISTS (SELECT 1 FROM host_software hsw WHERE hsw.software_id = s.id)
`
)
// Create a fresh swap table to populate with new counts. If a previous run left a partial swap table, drop it first.
@@ -2763,13 +2754,47 @@ func (ds *Datastore) SyncHostsSoftware(ctx context.Context, updatedAt time.Time)
return err
}
// Remove any unused software (those not in host_software).
if _, err := ds.writer(ctx).ExecContext(ctx, cleanupSoftwareStmt); err != nil {
return ctxerr.Wrap(ctx, err, "delete unused software")
// Remove any unused software (those not in host_software) in batches to reduce lock contention.
if err := ds.cleanupUnusedSoftware(ctx); err != nil {
return err
}
return nil
}
// cleanupUnusedSoftware deletes orphaned software rows (not referenced by any host) in batches.
func (ds *Datastore) cleanupUnusedSoftware(ctx context.Context) error {
// findUnusedSoftwareStmt finds software rows not referenced by any host and absent from software_host_counts.
// The NOT EXISTS check on host_software reduces (but does not fully prevent) the chance of deleting software that
// is mid-ingestion (inserted into software but not yet linked in host_software). In the unlikely event this happens,
// the next hourly ingestion cycle will re-create and re-link the software entry.
const findUnusedSoftwareStmt = `
SELECT s.id
FROM software s
LEFT JOIN software_host_counts shc ON s.id = shc.software_id
WHERE shc.software_id IS NULL
AND NOT EXISTS (SELECT 1 FROM host_software hsw WHERE hsw.software_id = s.id)
LIMIT ?
`
for {
var ids []uint
if err := sqlx.SelectContext(ctx, ds.writer(ctx), &ids, findUnusedSoftwareStmt, cleanupBatchSize); err != nil {
return ctxerr.Wrap(ctx, err, "find unused software for cleanup")
}
if len(ids) == 0 {
return nil
}
stmt, args, err := sqlx.In(`DELETE FROM software WHERE id IN (?)`, ids)
if err != nil {
return ctxerr.Wrap(ctx, err, "build delete unused software query")
}
if _, err := ds.writer(ctx).ExecContext(ctx, stmt, args...); err != nil {
return ctxerr.Wrap(ctx, err, "delete unused software batch")
}
}
}
func (ds *Datastore) CleanupSoftwareTitles(ctx context.Context) error {
var n int64
defer func(start time.Time) {