diff --git a/server/datastore/mysql/software_titles.go b/server/datastore/mysql/software_titles.go index 4ca53f6980..27f76e631c 100644 --- a/server/datastore/mysql/software_titles.go +++ b/server/datastore/mysql/software_titles.go @@ -9,6 +9,7 @@ import ( "time" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" + "github.com/fleetdm/fleet/v4/server/datastore/mysql/common_mysql" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/jmoiron/sqlx" ) @@ -221,18 +222,28 @@ func (ds *Datastore) ListSoftwareTitles( // the application logic. This is because we need to support MySQL 5.7 // and there's no good way to do an aggregation that builds a structure // (like a JSON) object for nested arrays. - getVersionsStmt, args, err := ds.selectSoftwareVersionsSQL( - titleIDs, - opt.TeamID, - tmFilter, - false, - ) - if err != nil { - return nil, 0, nil, ctxerr.Wrap(ctx, err, "build get versions stmt") - } + batchSize := 32000 var versions []fleet.SoftwareVersion - if err := sqlx.SelectContext(ctx, dbReader, &versions, getVersionsStmt, args...); err != nil { - return nil, 0, nil, ctxerr.Wrap(ctx, err, "get software versions") + err = common_mysql.BatchProcessSimple(titleIDs, batchSize, func(titleIDsToProcess []uint) error { + getVersionsStmt, args, err := ds.selectSoftwareVersionsSQL( + titleIDsToProcess, + opt.TeamID, + tmFilter, + false, + ) + if err != nil { + return ctxerr.Wrap(ctx, err, "build get versions stmt") + } + var versionsBatch []fleet.SoftwareVersion + if err := sqlx.SelectContext(ctx, dbReader, &versions, getVersionsStmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "get software versions") + } + versions = append(versions, versionsBatch...) + + return nil + }) + if err != nil { + return nil, 0, nil, err } // append matching versions to titles diff --git a/server/datastore/mysql/software_titles_test.go b/server/datastore/mysql/software_titles_test.go index d5a93f30b9..061ec4024c 100644 --- a/server/datastore/mysql/software_titles_test.go +++ b/server/datastore/mysql/software_titles_test.go @@ -3,6 +3,7 @@ package mysql import ( "context" "database/sql" + "fmt" "sort" "testing" "time" @@ -28,6 +29,7 @@ func TestSoftwareTitles(t *testing.T) { {"TeamFilterSoftwareTitles", testTeamFilterSoftwareTitles}, {"ListSoftwareTitlesInstallersOnly", testListSoftwareTitlesInstallersOnly}, {"ListSoftwareTitlesAvailableForInstallFilter", testListSoftwareTitlesAvailableForInstallFilter}, + {"ListSoftwareTitlesOverflow", testListSoftwareTitlesOverflow}, {"ListSoftwareTitlesAllTeams", testListSoftwareTitlesAllTeams}, {"UploadedSoftwareExists", testUploadedSoftwareExists}, {"ListSoftwareTitlesVulnerabilityFilters", testListSoftwareTitlesVulnerabilityFilters}, @@ -1181,6 +1183,47 @@ func testListSoftwareTitlesAvailableForInstallFilter(t *testing.T, ds *Datastore }, names) } +func testListSoftwareTitlesOverflow(t *testing.T, ds *Datastore) { + t.Skip("This test is too slow to run in CI") + ctx := context.Background() + + host := test.NewHost(t, ds, "host", "", "hostkey1", "hostuuid1", time.Now()) + host2 := test.NewHost(t, ds, "host2", "", "hostkey2", "hostuuid2", time.Now()) + + var software []fleet.Software + for i := uint(0); i < 40_000; i++ { + software = append(software, + fleet.Software{Name: fmt.Sprintf("%dname", i), Version: fmt.Sprintf("0.0.%d", i), Source: "deb_packages"}, + ) + // UpdateHostSoftware blows up on a similar placeholder limit if we don't break it up + if i == 20_000 { + _, err := ds.UpdateHostSoftware(ctx, host.ID, software) + require.NoError(t, err) + software = []fleet.Software{} + } + if i == 39_999 { + _, err := ds.UpdateHostSoftware(ctx, host2.ID, software) + require.NoError(t, err) + } + } + + require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) + + _, counts, _, err := ds.ListSoftwareTitles( + ctx, + fleet.SoftwareTitleListOptions{ + ListOptions: fleet.ListOptions{ + OrderKey: "name", + OrderDirection: fleet.OrderAscending, + }, + TeamID: nil, + }, + fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String(fleet.RoleAdmin)}}, + ) + require.NoError(t, err) + assert.EqualValues(t, 40000, counts) +} + func testListSoftwareTitlesAllTeams(t *testing.T, ds *Datastore) { ctx := context.Background()