From 1edd9f07bb07fc09b606ea369bd978fcc07f1b51 Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Tue, 26 Mar 2024 19:54:47 -0500 Subject: [PATCH] Update host details, list host filters, and MDM summary to include macOS declarations (#17866) Issue #17619 --------- Co-authored-by: Roberto Dip --- server/datastore/mysql/apple_mdm.go | 292 +++++++++++++++--- server/datastore/mysql/apple_mdm_test.go | 24 +- server/datastore/mysql/hosts.go | 60 ++-- server/datastore/mysql/labels_test.go | 5 + server/datastore/mysql/mdm_test.go | 372 ++++++++++++++++++++++- 5 files changed, 662 insertions(+), 91 deletions(-) diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index bf80e95f65..eb7143e2b0 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -352,16 +352,38 @@ COALESCE(detail, '') AS detail FROM host_mdm_apple_profiles WHERE +host_uuid = ? AND NOT (operation_type = '%s' AND COALESCE(status, '%s') IN('%s', '%s')) + +UNION ALL +SELECT +declaration_uuid AS profile_uuid, +declaration_name AS name, +declaration_identifier AS identifier, +-- internally, a NULL status implies that the cron needs to pick up +-- this profile, for the user that difference doesn't exist, the +-- profile is effectively pending. This is consistent with all our +-- aggregation functions. +COALESCE(status, '%s') AS status, +COALESCE(operation_type, '') AS operation_type, +COALESCE(detail, '') AS detail +FROM +host_mdm_apple_declarations +WHERE host_uuid = ? AND NOT (operation_type = '%s' AND COALESCE(status, '%s') IN('%s', '%s'))`, fleet.MDMDeliveryPending, fleet.MDMOperationTypeRemove, fleet.MDMDeliveryPending, fleet.MDMDeliveryVerifying, fleet.MDMDeliveryVerified, + fleet.MDMDeliveryPending, + fleet.MDMOperationTypeRemove, + fleet.MDMDeliveryPending, + fleet.MDMDeliveryVerifying, + fleet.MDMDeliveryVerified, ) var profiles []fleet.HostMDMAppleProfile - if err := sqlx.SelectContext(ctx, ds.reader(ctx), &profiles, stmt, hostUUID); err != nil { + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &profiles, stmt, hostUUID, hostUUID); err != nil { return nil, err } return profiles, nil @@ -2190,57 +2212,251 @@ func subqueryAppleProfileStatus(status fleet.MDMDeliveryStatus) (string, []any, return query, args, nil } +// subqueryAppleDeclarationStatus builds out the subquery for declaration status +func subqueryAppleDeclarationStatus() (string, []any, error) { + const declNamedStmt = ` + CASE WHEN EXISTS ( + SELECT + 1 + FROM + host_mdm_apple_declarations d1 + WHERE + h.uuid = d1.host_uuid + AND d1.status = :failed) THEN + 'declarations_failed' + WHEN EXISTS ( + SELECT + 1 + FROM + host_mdm_apple_declarations d2 + WHERE + h.uuid = d2.host_uuid + AND(d2.status IS NULL + OR d2.status = :pending) + AND NOT EXISTS ( + SELECT + 1 + FROM + host_mdm_apple_declarations d3 + WHERE + h.uuid = d3.host_uuid + AND d3.status = :failed)) THEN + 'declarations_pending' + WHEN EXISTS ( + SELECT + 1 + FROM + host_mdm_apple_declarations d4 + WHERE + h.uuid = d4.host_uuid + AND d4.status = :verifying + AND NOT EXISTS ( + SELECT + 1 + FROM + host_mdm_apple_declarations d5 + WHERE (h.uuid = d5.host_uuid + AND(d5.status IS NULL + OR d5.status IN(:pending, :failed))))) THEN + 'declarations_verifying' + WHEN EXISTS ( + SELECT + 1 + FROM + host_mdm_apple_declarations d6 + WHERE + h.uuid = d6.host_uuid + AND d6.status = :verified + AND NOT EXISTS ( + SELECT + 1 + FROM + host_mdm_apple_declarations d7 + WHERE (h.uuid = d7.host_uuid + AND(d7.status IS NULL + OR d7.status IN(:pending, :failed, :verifying))))) THEN + 'declarations_verified' + ELSE + '' + END` + + // TODO: do we need to differentiate between install and remove? + arg := map[string]any{ + // "install": fleet.MDMOperationTypeInstall, + // "remove": fleet.MDMOperationTypeRemove, + "verifying": fleet.MDMDeliveryVerifying, + "failed": fleet.MDMDeliveryFailed, + "verified": fleet.MDMDeliveryVerified, + "pending": fleet.MDMDeliveryPending, + } + query, args, err := sqlx.Named(declNamedStmt, arg) + if err != nil { + return "", nil, fmt.Errorf("subqueryAppleDeclarationStatus: %w", err) + } + + return query, args, nil +} + +func subqueryOSSettingsStatusMac() (string, []any, error) { + var profArgs []any + profFailed, profFailedArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryFailed) + if err != nil { + return "", nil, err + } + profArgs = append(profArgs, profFailedArgs...) + + profPending, profPendingArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryPending) + if err != nil { + return "", nil, err + } + profArgs = append(profArgs, profPendingArgs...) + + profVerifying, profVerifyingArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryVerifying) + if err != nil { + return "", nil, err + } + profArgs = append(profArgs, profVerifyingArgs...) + + profVerified, profVerifiedArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryVerified) + if err != nil { + return "", nil, err + } + profArgs = append(profArgs, profVerifiedArgs...) + + profStmt := fmt.Sprintf(` + CASE WHEN EXISTS (%s) THEN + 'profiles_failed' + WHEN EXISTS (%s) THEN + 'profiles_pending' + WHEN EXISTS (%s) THEN + 'profiles_verifying' + WHEN EXISTS (%s) THEN + 'profiles_verified' + ELSE + '' + END`, + profFailed, + profPending, + profVerifying, + profVerified, + ) + + declStmt, declArgs, err := subqueryAppleDeclarationStatus() + if err != nil { + return "", nil, err + } + + stmt := fmt.Sprintf(` + CASE (%s) + WHEN 'profiles_failed' THEN + 'failed' + WHEN 'profiles_pending' THEN ( + CASE (%s) + WHEN 'declarations_failed' THEN + 'failed' + ELSE + 'pending' + END) + WHEN 'profiles_verifying' THEN ( + CASE (%s) + WHEN 'declarations_failed' THEN + 'failed' + WHEN 'declarations_pending' THEN + 'pending' + ELSE + 'verifying' + END) + WHEN 'profiles_verified' THEN ( + CASE (%s) + WHEN 'declarations_failed' THEN + 'failed' + WHEN 'declarations_pending' THEN + 'pending' + WHEN 'declarations_verifying' THEN + 'verifying' + ELSE + 'verified' + END) + ELSE + REPLACE((%s), 'declarations_', '') + END`, profStmt, declStmt, declStmt, declStmt, declStmt) + + args := append(profArgs, declArgs...) + args = append(args, declArgs...) + args = append(args, declArgs...) + args = append(args, declArgs...) + + // FIXME(roberto): we found issues in MySQL 5.7.17 (only that version, + // which we must support for now) with prepared statements on this + // query. The results returned by the DB were always different what + // expected unless the arguments are inlined in the query. + // + // We decided to do this given: + // + // - The time constraints we were given to develop DDM + // - The fact that all the variables in this query are really strings managed by us + // - The imminent deprecation of MySQL 5.7 + return fmt.Sprintf(strings.Replace(stmt, "?", "'%s'", -1), args...), []any{}, nil +} + func (ds *Datastore) GetMDMAppleProfilesSummary(ctx context.Context, teamID *uint) (*fleet.MDMProfilesSummary, error) { - var args []interface{} - - subqueryFailed, subqueryFailedArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryFailed) + subquery, args, err := subqueryOSSettingsStatusMac() if err != nil { - return nil, ctxerr.Wrap(ctx, err, "building failed subquery") + return nil, ctxerr.Wrap(ctx, err, "building os settings subquery") } - args = append(args, subqueryFailedArgs...) - - subqueryPending, subqueryPendingArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryPending) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "building pending subquery") - } - args = append(args, subqueryPendingArgs...) - - subqueryVerifying, subqueryVerifyingArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryVerifying) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "building verifying subquery") - } - args = append(args, subqueryVerifyingArgs...) - - subqueryVerified, subqueryVerifiedArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryVerified) - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "building verified subquery") - } - args = append(args, subqueryVerifiedArgs...) sqlFmt := ` - SELECT - COUNT(CASE WHEN EXISTS (%s) THEN 1 END) AS failed, - COUNT(CASE WHEN EXISTS (%s) THEN 1 END) AS pending, - COUNT(CASE WHEN EXISTS (%s) THEN 1 END) AS verifying, - COUNT(CASE WHEN EXISTS (%s) THEN 1 END) AS verified - FROM - hosts h - WHERE - h.platform = 'darwin' AND %s` +SELECT + %s as status, + COUNT(id) as count +FROM + hosts h +GROUP BY status, platform, team_id HAVING platform = 'darwin' AND status IN (?, ?, ?, ?) AND %s` - teamFilter := "h.team_id IS NULL" + args = append(args, fleet.MDMDeliveryFailed, fleet.MDMDeliveryPending, fleet.MDMDeliveryVerifying, fleet.MDMDeliveryVerified) + + teamFilter := "team_id IS NULL" if teamID != nil && *teamID > 0 { - teamFilter = "h.team_id = ?" + teamFilter = "team_id = ?" args = append(args, *teamID) } - stmt := fmt.Sprintf(sqlFmt, subqueryFailed, subqueryPending, subqueryVerifying, subqueryVerified, teamFilter) - var res fleet.MDMProfilesSummary - err = sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, args...) + stmt := fmt.Sprintf(sqlFmt, subquery, teamFilter) + + var dest []struct { + Count uint `db:"count"` + Status string `db:"status"` + } + + err = sqlx.SelectContext(ctx, ds.reader(ctx), &dest, stmt, args...) if err != nil { return nil, err } + byStatus := make(map[string]uint) + for _, s := range dest { + if _, ok := byStatus[s.Status]; ok { + return nil, fmt.Errorf("duplicate status %s", s.Status) + } + byStatus[s.Status] = s.Count + } + + var res fleet.MDMProfilesSummary + for s, c := range byStatus { + switch fleet.MDMDeliveryStatus(s) { + case fleet.MDMDeliveryFailed: + res.Failed = c + case fleet.MDMDeliveryPending: + res.Pending = c + case fleet.MDMDeliveryVerifying: + res.Verifying = c + case fleet.MDMDeliveryVerified: + res.Verified = c + default: + return nil, fmt.Errorf("unknown status %s", s) + } + } + return &res, nil } diff --git a/server/datastore/mysql/apple_mdm_test.go b/server/datastore/mysql/apple_mdm_test.go index 6eed346a69..d989cc2719 100644 --- a/server/datastore/mysql/apple_mdm_test.go +++ b/server/datastore/mysql/apple_mdm_test.go @@ -1787,13 +1787,19 @@ func testAggregateMacOSSettingsStatusWithFileVault(t *testing.T, ds *Datastore) expectedIDs = append(expectedIDs, h.ID) } - gotHosts, err := ds.ListHosts(ctx, fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String("admin")}}, fleet.HostListOptions{MacOSSettingsFilter: status, TeamFilter: teamID}) + gotHosts, err := ds.ListHosts( + ctx, + fleet.TeamFilter{User: &fleet.User{GlobalRole: ptr.String("admin")}}, + fleet.HostListOptions{MacOSSettingsFilter: status, TeamFilter: teamID}, + ) gotIDs := []uint{} for _, h := range gotHosts { gotIDs = append(gotIDs, h.ID) } - return assert.NoError(t, err) && assert.Len(t, gotHosts, len(expected)) && assert.ElementsMatch(t, expectedIDs, gotIDs) + return assert.NoError(t, err) && + assert.Len(t, gotHosts, len(expected)) && + assert.ElementsMatch(t, expectedIDs, gotIDs) } var hosts []*fleet.Host @@ -2615,7 +2621,7 @@ func TestMDMAppleFileVaultSummary(t *testing.T) { allProfilesSummary, err = ds.GetMDMAppleProfilesSummary(ctx, nil) require.NoError(t, err) - require.NotNil(t, fvProfileSummary) + require.NotNil(t, allProfilesSummary) require.Equal(t, uint(2), allProfilesSummary.Pending) require.Equal(t, uint(0), allProfilesSummary.Failed) require.Equal(t, uint(1), allProfilesSummary.Verifying) @@ -2641,7 +2647,7 @@ func TestMDMAppleFileVaultSummary(t *testing.T) { allProfilesSummary, err = ds.GetMDMAppleProfilesSummary(ctx, nil) require.NoError(t, err) - require.NotNil(t, fvProfileSummary) + require.NotNil(t, allProfilesSummary) require.Equal(t, uint(2), allProfilesSummary.Pending) require.Equal(t, uint(0), allProfilesSummary.Failed) require.Equal(t, uint(1), allProfilesSummary.Verifying) @@ -2669,7 +2675,7 @@ func TestMDMAppleFileVaultSummary(t *testing.T) { allProfilesSummary, err = ds.GetMDMAppleProfilesSummary(ctx, nil) require.NoError(t, err) - require.NotNil(t, fvProfileSummary) + require.NotNil(t, allProfilesSummary) require.Equal(t, uint(2), allProfilesSummary.Pending) require.Equal(t, uint(0), allProfilesSummary.Failed) require.Equal(t, uint(1), allProfilesSummary.Verifying) @@ -2691,7 +2697,7 @@ func TestMDMAppleFileVaultSummary(t *testing.T) { allProfilesSummary, err = ds.GetMDMAppleProfilesSummary(ctx, nil) require.NoError(t, err) - require.NotNil(t, fvProfileSummary) + require.NotNil(t, allProfilesSummary) require.Equal(t, uint(2), allProfilesSummary.Pending) require.Equal(t, uint(1), allProfilesSummary.Failed) require.Equal(t, uint(1), allProfilesSummary.Verifying) @@ -2713,7 +2719,7 @@ func TestMDMAppleFileVaultSummary(t *testing.T) { allProfilesSummary, err = ds.GetMDMAppleProfilesSummary(ctx, nil) require.NoError(t, err) - require.NotNil(t, fvProfileSummary) + require.NotNil(t, allProfilesSummary) require.Equal(t, uint(3), allProfilesSummary.Pending) require.Equal(t, uint(1), allProfilesSummary.Failed) require.Equal(t, uint(1), allProfilesSummary.Verifying) @@ -2743,7 +2749,7 @@ func TestMDMAppleFileVaultSummary(t *testing.T) { allProfilesSummary, err = ds.GetMDMAppleProfilesSummary(ctx, &tm.ID) require.NoError(t, err) - require.NotNil(t, fvProfileSummary) + require.NotNil(t, allProfilesSummary) require.Equal(t, uint(0), allProfilesSummary.Pending) require.Equal(t, uint(0), allProfilesSummary.Failed) require.Equal(t, uint(1), allProfilesSummary.Verifying) @@ -2769,7 +2775,7 @@ func TestMDMAppleFileVaultSummary(t *testing.T) { allProfilesSummary, err = ds.GetMDMAppleProfilesSummary(ctx, &tm.ID) require.NoError(t, err) - require.NotNil(t, fvProfileSummary) + require.NotNil(t, allProfilesSummary) require.Equal(t, uint(0), allProfilesSummary.Pending) require.Equal(t, uint(0), allProfilesSummary.Failed) require.Equal(t, uint(0), allProfilesSummary.Verifying) diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index df2f4395bc..5c78af5b15 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -1210,34 +1210,22 @@ func filterHostsByMacOSSettingsStatus(sql string, opt fleet.HostListOptions, par return sql, params, nil } - newSQL := "" + whereStatus := "" + // macOS settings filter is not compatible with the "all teams" option so append the "no + // team" filter here (note that filterHostsByTeam applies the "no team" filter if TeamFilter == 0) if opt.TeamFilter == nil { - // macOS settings filter is not compatible with the "all teams" option so append the "no - // team" filter here (note that filterHostsByTeam applies the "no team" filter if TeamFilter == 0) - newSQL += ` AND h.team_id IS NULL` + whereStatus += ` AND h.team_id IS NULL` } - var subquery string - var subqueryParams []any - var err error - switch opt.MacOSSettingsFilter { - case fleet.OSSettingsFailed: - subquery, subqueryParams, err = subqueryAppleProfileStatus(fleet.MDMDeliveryFailed) - case fleet.OSSettingsPending: - subquery, subqueryParams, err = subqueryAppleProfileStatus(fleet.MDMDeliveryPending) - case fleet.OSSettingsVerifying: - subquery, subqueryParams, err = subqueryAppleProfileStatus(fleet.MDMDeliveryVerifying) - case fleet.OSSettingsVerified: - subquery, subqueryParams, err = subqueryAppleProfileStatus(fleet.MDMDeliveryVerified) - } + subqueryStatus, paramsStatus, err := subqueryOSSettingsStatusMac() if err != nil { - return "", nil, fmt.Errorf("building subquery for %s filter: %w", opt.MacOSSettingsFilter, err) - } - if subquery != "" { - newSQL += fmt.Sprintf(` AND EXISTS (%s)`, subquery) + return "", nil, err } - return sql + newSQL, append(params, subqueryParams...), nil + whereStatus += fmt.Sprintf(` AND %s = ?`, subqueryStatus) + paramsStatus = append(paramsStatus, opt.MacOSSettingsFilter) + + return sql + whereStatus, append(params, paramsStatus...), nil } func filterHostsByMacOSDiskEncryptionStatus(sql string, opt fleet.HostListOptions, params []interface{}) (string, []interface{}) { @@ -1285,30 +1273,16 @@ func (ds *Datastore) filterHostsByOSSettingsStatus(sql string, opt fleet.HostLis sqlFmt += ` AND h.team_id IS NULL` } var whereMacOS, whereWindows string - sqlFmt += ` AND ((h.platform = 'windows' AND (%s)) OR (h.platform = 'darwin' AND (%s)))` + sqlFmt += ` +AND ((h.platform = 'windows' AND (%s)) +OR (h.platform = 'darwin' AND (%s)))` - // construct the WHERE for macOS - var subqueryMacOS string - var paramsMacOS []interface{} - var err error - switch opt.OSSettingsFilter { - case fleet.OSSettingsFailed: - subqueryMacOS, paramsMacOS, err = subqueryAppleProfileStatus(fleet.MDMDeliveryFailed) - case fleet.OSSettingsPending: - subqueryMacOS, paramsMacOS, err = subqueryAppleProfileStatus(fleet.MDMDeliveryPending) - case fleet.OSSettingsVerifying: - subqueryMacOS, paramsMacOS, err = subqueryAppleProfileStatus(fleet.MDMDeliveryVerifying) - case fleet.OSSettingsVerified: - subqueryMacOS, paramsMacOS, err = subqueryAppleProfileStatus(fleet.MDMDeliveryVerified) - } + whereMacOS, paramsMacOS, err := subqueryOSSettingsStatusMac() if err != nil { - return "", nil, fmt.Errorf("building subquery for %s filter: %w", opt.OSSettingsFilter, err) - } - if subqueryMacOS != "" { - whereMacOS = "EXISTS (" + subqueryMacOS + ")" - } else { - whereMacOS = "FALSE" + return "", nil, err } + whereMacOS += ` = ?` + paramsMacOS = append(paramsMacOS, opt.OSSettingsFilter) // construct the WHERE for windows whereWindows = `hmdm.name = ? AND hmdm.enrolled = 1 AND hmdm.is_server = 0` diff --git a/server/datastore/mysql/labels_test.go b/server/datastore/mysql/labels_test.go index eb1ddcd9fa..ea9d455c9a 100644 --- a/server/datastore/mysql/labels_test.go +++ b/server/datastore/mysql/labels_test.go @@ -364,6 +364,7 @@ func testLabelsListHostsInLabelAndStatus(t *testing.T, db *Datastore) { NodeKey: ptr.String("1"), UUID: "1", Hostname: "foo.local", + Platform: "darwin", }) require.NoError(t, err) @@ -377,6 +378,7 @@ func testLabelsListHostsInLabelAndStatus(t *testing.T, db *Datastore) { NodeKey: ptr.String("2"), UUID: "2", Hostname: "bar.local", + Platform: "darwin", }) require.NoError(t, err) h3, err := db.NewHost(context.Background(), &fleet.Host{ @@ -388,6 +390,7 @@ func testLabelsListHostsInLabelAndStatus(t *testing.T, db *Datastore) { NodeKey: ptr.String("3"), UUID: "3", Hostname: "baz.local", + Platform: "darwin", }) require.NoError(t, err) @@ -427,6 +430,7 @@ func testLabelsListHostsInLabelAndTeamFilter(deferred bool, t *testing.T, db *Da NodeKey: ptr.String("1"), UUID: "1", Hostname: "foo.local", + Platform: "darwin", }) require.Nil(t, err) @@ -440,6 +444,7 @@ func testLabelsListHostsInLabelAndTeamFilter(deferred bool, t *testing.T, db *Da NodeKey: ptr.String("2"), UUID: "2", Hostname: "bar.local", + Platform: "darwin", }) require.Nil(t, err) diff --git a/server/datastore/mysql/mdm_test.go b/server/datastore/mysql/mdm_test.go index f5784bdb10..b962cf8f68 100644 --- a/server/datastore/mysql/mdm_test.go +++ b/server/datastore/mysql/mdm_test.go @@ -7,6 +7,7 @@ import ( "fmt" "sort" "strconv" + "strings" "testing" "time" @@ -43,12 +44,12 @@ func TestMDMShared(t *testing.T) { {"TestMDMEULA", testMDMEULA}, {"TestGetHostCertAssociationsToExpire", testSCEPRenewalHelpers}, {"TestSCEPRenewalHelpers", testSCEPRenewalHelpers}, + {"TestMDMProfilesSummaryAndHostFilters", testMDMProfilesSummaryAndHostFilters}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { defer TruncateTables(t, ds) - c.fn(t, ds) }) } @@ -3330,3 +3331,372 @@ func testSCEPRenewalHelpers(t *testing.T, ds *Datastore) { require.NoError(t, err) checkSCEPRenew(assocs[0], nil) } + +func testMDMProfilesSummaryAndHostFilters(t *testing.T, ds *Datastore) { + // TODO: Expand this test to include: + // - more scenarios for windows + // - disk encryption (mac and windows) + // - more scenarios for labels + + ctx := context.Background() + + checkSummaryWindows := func(t *testing.T, teamID *uint, expected fleet.MDMProfilesSummary) { + ps, err := ds.GetMDMWindowsProfilesSummary(ctx, teamID) + require.NoError(t, err) + require.NotNil(t, ps) + require.Equal(t, expected, *ps) + } + + checkSummaryMac := func(t *testing.T, teamID *uint, expected fleet.MDMProfilesSummary) { + ps, err := ds.GetMDMAppleProfilesSummary(ctx, teamID) + require.NoError(t, err) + require.NotNil(t, ps) + require.Equal(t, expected, *ps) + } + + checkListHostsFilterOSSettings := func(t *testing.T, teamID *uint, status fleet.OSSettingsStatus, expectedIDs []uint) { + gotHosts, err := ds.ListHosts(ctx, fleet.TeamFilter{User: test.UserAdmin}, fleet.HostListOptions{TeamFilter: teamID, OSSettingsFilter: status}) + require.NoError(t, err) + if len(expectedIDs) != len(gotHosts) { + gotIDs := make([]uint, len(gotHosts)) + for _, h := range gotHosts { + gotIDs = append(gotIDs, h.ID) + } + require.Len(t, gotHosts, len(expectedIDs), fmt.Sprintf("status: %s expected: %v got: %v", status, expectedIDs, gotIDs)) + + } + for _, h := range gotHosts { + require.Contains(t, expectedIDs, h.ID) + } + + count, err := ds.CountHosts(ctx, fleet.TeamFilter{User: test.UserAdmin}, fleet.HostListOptions{TeamFilter: teamID, OSSettingsFilter: status}) + require.NoError(t, err) + require.Equal(t, len(expectedIDs), count, "status: %s", status) + } + + type hostIDsByProfileStatus map[fleet.MDMDeliveryStatus][]uint + + checkExpected := func(t *testing.T, teamID *uint, ep hostIDsByProfileStatus) { + expectSummaryWindows := map[fleet.MDMDeliveryStatus]uint{} + expectSummaryMac := map[fleet.MDMDeliveryStatus]uint{} + for status, ids := range ep { + if len(ids) > 0 { + for _, id := range ids { + if id < 5 { + expectSummaryWindows[status]++ + } else { + expectSummaryMac[status]++ + } + } + } + } + checkSummaryMac(t, teamID, fleet.MDMProfilesSummary{ + Pending: expectSummaryMac[fleet.MDMDeliveryPending], + Failed: expectSummaryMac[fleet.MDMDeliveryFailed], + Verifying: expectSummaryMac[fleet.MDMDeliveryVerifying], + Verified: expectSummaryMac[fleet.MDMDeliveryVerified], + }) + + checkSummaryWindows(t, teamID, fleet.MDMProfilesSummary{ + Pending: expectSummaryWindows[fleet.MDMDeliveryPending], + Failed: expectSummaryWindows[fleet.MDMDeliveryFailed], + Verifying: expectSummaryWindows[fleet.MDMDeliveryVerifying], + Verified: expectSummaryWindows[fleet.MDMDeliveryVerified], + }) + + checkListHostsFilterOSSettings(t, teamID, fleet.OSSettingsVerified, ep[fleet.MDMDeliveryVerified]) + checkListHostsFilterOSSettings(t, teamID, fleet.OSSettingsVerifying, ep[fleet.MDMDeliveryVerifying]) + checkListHostsFilterOSSettings(t, teamID, fleet.OSSettingsFailed, ep[fleet.MDMDeliveryFailed]) + checkListHostsFilterOSSettings(t, teamID, fleet.OSSettingsPending, ep[fleet.MDMDeliveryPending]) + } + + // checkWinHostProfiles := func(t *testing.T, hostUUID string, statusByProfUUID map[string]string) { + // profs, err := ds.GetHostMDMWindowsProfiles(ctx, hostUUID) + // require.NoError(t, err) + // require.Len(t, profs, len(statusByProfUUID)) + // for _, prof := range profs { + // ep, ok := statusByProfUUID[prof.ProfileUUID] + // require.True(t, ok) + // require.Equal(t, ep, prof.Status) + // } + // } + + checkMacHostProfiles := func(t *testing.T, hostUUID string, statusByProfUUID map[string]string) { + profs, err := ds.GetHostMDMAppleProfiles(ctx, hostUUID) + require.NoError(t, err) + require.Len(t, profs, len(statusByProfUUID)) + for _, prof := range profs { + ep, ok := statusByProfUUID[prof.ProfileUUID] + require.True(t, ok) + require.NotNil(t, prof.Status) + require.Equal(t, fleet.MDMDeliveryStatus(ep), *prof.Status) + } + } + + upsertHostProfileStatus := func(t *testing.T, hostUUID string, profUUID string, status *fleet.MDMDeliveryStatus) { + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + var table string + var profType string + switch { + case strings.HasPrefix(profUUID, "a"): + table = "host_mdm_apple_profiles" + profType = "profile" + case strings.HasPrefix(profUUID, "w"): + table = "host_mdm_windows_profiles" + profType = "profile" + case strings.HasPrefix(profUUID, "d"): + table = "host_mdm_apple_declarations" + profType = "declaration" + default: + require.FailNow(t, "unknown profile type") + } + stmt := fmt.Sprintf(`INSERT INTO %s (host_uuid, %s_uuid, status) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE status = ?`, table, profType) + _, err := q.ExecContext(ctx, stmt, hostUUID, profUUID, status, status) + if err != nil { + require.NoError(t, err) + return err + } + stmt = fmt.Sprintf(`UPDATE %s SET operation_type = ? WHERE host_uuid = ? AND %s_uuid = ?`, table, profType) + _, err = q.ExecContext(ctx, stmt, fleet.MDMOperationTypeInstall, hostUUID, profUUID) + require.NoError(t, err) + return err + }) + } + + cleanupTables := func(t *testing.T) { + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM host_mdm_windows_profiles`) + return err + }) + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM host_mdm_apple_profiles`) + return err + }) + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM host_mdm_apple_declarations`) + return err + }) + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM host_disk_encryption_keys`) + return err + }) + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM host_disks`) + return err + }) + } + + // updateHostDisks := func(t *testing.T, hostID uint, encrypted bool, updated_at time.Time) { + // ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + // stmt := `UPDATE host_disks SET encrypted = ?, updated_at = ? where host_id = ?` + // _, err := q.ExecContext(ctx, stmt, encrypted, updated_at, hostID) + // return err + // }) + // } + + // Create some hosts + var hosts []*fleet.Host + macHostsByID := make(map[uint]*fleet.Host, 5) + winHostsByID := make(map[uint]*fleet.Host, 5) + for i := 0; i < 10; i++ { + p := "windows" + if i >= 5 { + p = "darwin" + } + u := uuid.New().String() + h, err := ds.NewHost(ctx, &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now(), + NodeKey: &u, + UUID: u, + Hostname: u, + Platform: p, + }) + require.NoError(t, err) + require.NotNil(t, h) + hosts = append(hosts, h) + if p == "darwin" { + macHostsByID[h.ID] = h + } else { + winHostsByID[h.ID] = h + } + + require.NoError(t, ds.SetOrUpdateMDMData(ctx, h.ID, false, true, "https://example.com", false, fleet.WellKnownMDMFleet, "")) + } + + checkExpected(t, nil, nil) + + upsertHostProfileStatus(t, hosts[0].UUID, "w1", &fleet.MDMDeliveryPending) + + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + }) + + // add some mac profiles with different statuses + upsertHostProfileStatus(t, hosts[9].UUID, "a1", &fleet.MDMDeliveryFailed) + upsertHostProfileStatus(t, hosts[9].UUID, "a2", &fleet.MDMDeliveryPending) + upsertHostProfileStatus(t, hosts[9].UUID, "a3", &fleet.MDMDeliveryVerifying) + upsertHostProfileStatus(t, hosts[9].UUID, "a4", &fleet.MDMDeliveryVerified) + + // add some mac declarations with different statuses + upsertHostProfileStatus(t, hosts[9].UUID, "d1", &fleet.MDMDeliveryFailed) + upsertHostProfileStatus(t, hosts[9].UUID, "d2", &fleet.MDMDeliveryPending) + upsertHostProfileStatus(t, hosts[9].UUID, "d3", &fleet.MDMDeliveryVerifying) + upsertHostProfileStatus(t, hosts[9].UUID, "d4", &fleet.MDMDeliveryVerified) + + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryFailed: []uint{hosts[9].ID}, + }) + expectedHostProfiles := map[string]string{ + "a1": "failed", + "a2": "pending", + "a3": "verifying", + "a4": "verified", + "d1": "failed", + "d2": "pending", + "d3": "verifying", + "d4": "verified", + } + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set failed mac profile to pending, still failed because of failed declaration + upsertHostProfileStatus(t, hosts[9].UUID, "a1", &fleet.MDMDeliveryPending) + expectedHostProfiles["a1"] = "pending" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryFailed: []uint{hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set failed mac declaration to pending, now host stsatus is pending + upsertHostProfileStatus(t, hosts[9].UUID, "d1", &fleet.MDMDeliveryPending) + expectedHostProfiles["d1"] = "pending" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID, hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set pending mac declaration to failed, host status is now failed + upsertHostProfileStatus(t, hosts[9].UUID, "d2", &fleet.MDMDeliveryFailed) + expectedHostProfiles["d2"] = "failed" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryFailed: []uint{hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set failed mac declaration to verifying, host status is now pending + upsertHostProfileStatus(t, hosts[9].UUID, "d2", &fleet.MDMDeliveryVerifying) + expectedHostProfiles["d2"] = "verifying" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID, hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set pending mac profiles to verifying, host status is still pending because d1 is still pending + upsertHostProfileStatus(t, hosts[9].UUID, "a1", &fleet.MDMDeliveryVerifying) + expectedHostProfiles["a1"] = "verifying" + upsertHostProfileStatus(t, hosts[9].UUID, "a2", &fleet.MDMDeliveryVerifying) + expectedHostProfiles["a2"] = "verifying" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID, hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set pending mac declarations to verifying, host status is now verifying + upsertHostProfileStatus(t, hosts[9].UUID, "d1", &fleet.MDMDeliveryVerifying) + expectedHostProfiles["d1"] = "verifying" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryVerifying: []uint{hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set a mac profile to failed, host status is now failed + upsertHostProfileStatus(t, hosts[9].UUID, "a1", &fleet.MDMDeliveryFailed) + expectedHostProfiles["a1"] = "failed" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryFailed: []uint{hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set mac profiles to verified, host status is now verifying because declarations are still + // verifying + upsertHostProfileStatus(t, hosts[9].UUID, "a1", &fleet.MDMDeliveryVerified) + expectedHostProfiles["a1"] = "verified" + upsertHostProfileStatus(t, hosts[9].UUID, "a2", &fleet.MDMDeliveryVerified) + expectedHostProfiles["a2"] = "verified" + upsertHostProfileStatus(t, hosts[9].UUID, "a3", &fleet.MDMDeliveryVerified) + expectedHostProfiles["a3"] = "verified" + upsertHostProfileStatus(t, hosts[9].UUID, "a4", &fleet.MDMDeliveryVerified) + expectedHostProfiles["a4"] = "verified" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryVerifying: []uint{hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set mac declarations to verified, host status is now verified + upsertHostProfileStatus(t, hosts[9].UUID, "d1", &fleet.MDMDeliveryVerified) + expectedHostProfiles["d1"] = "verified" + upsertHostProfileStatus(t, hosts[9].UUID, "d2", &fleet.MDMDeliveryVerified) + expectedHostProfiles["d2"] = "verified" + upsertHostProfileStatus(t, hosts[9].UUID, "d3", &fleet.MDMDeliveryVerified) + expectedHostProfiles["d3"] = "verified" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryVerified: []uint{hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // set a mac declaration to nil, host status is now pending + upsertHostProfileStatus(t, hosts[9].UUID, "d1", nil) + expectedHostProfiles["d1"] = "pending" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID, hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // works as expected if we remove mac declarations + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM host_mdm_apple_declarations`) + return err + }) + delete(expectedHostProfiles, "d1") + delete(expectedHostProfiles, "d2") + delete(expectedHostProfiles, "d3") + delete(expectedHostProfiles, "d4") + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + fleet.MDMDeliveryVerified: []uint{hosts[9].ID}, // all profiles were verified + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // works as expected if we remove mac profiles + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + _, err := q.ExecContext(ctx, `DELETE FROM host_mdm_apple_profiles`) + return err + }) + delete(expectedHostProfiles, "a1") + delete(expectedHostProfiles, "a2") + delete(expectedHostProfiles, "a3") + delete(expectedHostProfiles, "a4") + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + // works as expected if declarations but no profiles + upsertHostProfileStatus(t, hosts[9].UUID, "d1", &fleet.MDMDeliveryPending) + expectedHostProfiles["d1"] = "pending" + checkExpected(t, nil, hostIDsByProfileStatus{ + fleet.MDMDeliveryPending: []uint{hosts[0].ID, hosts[9].ID}, + }) + checkMacHostProfiles(t, hosts[9].UUID, expectedHostProfiles) + + cleanupTables(t) +}