diff --git a/changes/46243-join-mdm-for-missing-status b/changes/46243-join-mdm-for-missing-status new file mode 100644 index 0000000000..0306dd03c5 --- /dev/null +++ b/changes/46243-join-mdm-for-missing-status @@ -0,0 +1 @@ +* Fixed an issue where Missing hosts filter and dashboard card incorrectly reported iOS, iPadOS, and Android hosts. \ No newline at end of file diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 2dac8c2673..699ec02cda 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -1431,6 +1431,11 @@ func (ds *Datastore) applyHostFilters( batchScriptExecutionJoin, batchScriptExecutionFilter, whereParams = ds.getBatchExecutionFilters(whereParams, opt) } + hostMDMSeenJoin := "" + if opt.StatusFilter.IsValid() { + hostMDMSeenJoin = hostMDMSeenTimeJoin + } + var depStatusFilter string wantFailedDEP := ptr.ValOrZero(opt.DEPProfileErrorFilter) wantDepResp := string(ptr.ValOrZero(opt.DEPAssignProfileResponseFilter)) @@ -1462,6 +1467,7 @@ func (ds *Datastore) applyHostFilters( %s %s %s + %s %s WHERE TRUE AND %s AND %s AND %s AND %s AND %s %s `, @@ -1481,6 +1487,7 @@ func (ds *Datastore) applyHostFilters( mdmRecoveryLockStatusJoin, mdmAndroidProfilesStatusJoin, batchScriptExecutionJoin, + hostMDMSeenJoin, // Conditions ds.whereFilterHostsByTeams(filter, "h"), @@ -1652,6 +1659,19 @@ func filterHostsByPolicy(sql string, opt fleet.HostListOptions, params []interfa return sql, params } +// hostMDMSeenTimeJoin joins on nano enrollment so that the effective last-seen +// time can fall back to the Apple MDM protocol's last_seen_at +// for hosts that never check in via osquery (ios/ipados). +// It uses a dedicated alias (nes) to avoid colliding with the connected-to-Fleet join (ne) +const hostMDMSeenTimeJoin = ` + LEFT JOIN nano_enrollments nes ON nes.id = h.uuid AND nes.type IN ('Device', 'User Enrollment (Device)')` + +// hostEffectiveLastSeenExpr is the effective "last seen" time for a host: the greatest of the osquery +// seen_time and the MDM last_seen_at, then detail_updated_at (treating the Never sentinel as null), +// then created_at. +// Requires hostMDMSeenTimeJoin (alias nes) and the host_seen_times join (alias hst) to be present. +const hostEffectiveLastSeenExpr = `COALESCE(GREATEST(COALESCE(hst.seen_time, nes.last_seen_at), COALESCE(nes.last_seen_at, hst.seen_time)), NULLIF(h.detail_updated_at, '` + server.NeverTimestamp + `'), h.created_at)` + func filterHostsByStatus(now time.Time, sql string, opt fleet.HostListOptions, params []interface{}) (string, []interface{}) { switch opt.StatusFilter { case fleet.StatusNew: @@ -1664,7 +1684,8 @@ func filterHostsByStatus(now time.Time, sql string, opt fleet.HostListOptions, p sql += fmt.Sprintf("AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(h.distributed_interval, h.config_tls_refresh) + %d SECOND) <= ?", fleet.OnlineIntervalBuffer) params = append(params, now) case fleet.StatusMIA, fleet.StatusMissing: - sql += "AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) <= ? AND (hmdm.enrollment_status IS NULL OR hmdm.enrollment_status != 'Pending')" + // This must stay in sync with the missing_30_days_count computation in GenerateHostStatusStatistics. + sql += "AND DATE_ADD(" + hostEffectiveLastSeenExpr + ", INTERVAL 30 DAY) <= ? AND (hmdm.enrollment_status IS NULL OR hmdm.enrollment_status != 'Pending')" params = append(params, now) } return sql, params @@ -2140,15 +2161,15 @@ func (ds *Datastore) GenerateHostStatusStatistics(ctx context.Context, filter fl sqlStatement := fmt.Sprintf(` SELECT COUNT(*) total, - COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) <= ? AND (hmdm.enrollment_status IS NULL OR hmdm.enrollment_status != 'Pending') THEN 1 ELSE 0 END), 0) mia, - COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) <= ? AND (hmdm.enrollment_status IS NULL OR hmdm.enrollment_status != 'Pending') THEN 1 ELSE 0 END), 0) missing_30_days_count, + COALESCE(SUM(CASE WHEN DATE_ADD(`+hostEffectiveLastSeenExpr+`, INTERVAL 30 DAY) <= ? AND (hmdm.enrollment_status IS NULL OR hmdm.enrollment_status != 'Pending') THEN 1 ELSE 0 END), 0) mia, + COALESCE(SUM(CASE WHEN DATE_ADD(`+hostEffectiveLastSeenExpr+`, INTERVAL 30 DAY) <= ? AND (hmdm.enrollment_status IS NULL OR hmdm.enrollment_status != 'Pending') THEN 1 ELSE 0 END), 0) missing_30_days_count, COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) <= ? THEN 1 ELSE 0 END), 0) offline, COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) > ? THEN 1 ELSE 0 END), 0) online, COALESCE(SUM(CASE WHEN DATE_ADD(h.created_at, INTERVAL 1 DAY) >= ? THEN 1 ELSE 0 END), 0) new, COALESCE(SUM(CASE WHEN hdep.assign_profile_response IN (%s, %s) THEN 1 ELSE 0 END), 0) dep_assign_error_count, %s FROM hosts h - LEFT JOIN host_seen_times hst ON (h.id = hst.host_id) + LEFT JOIN host_seen_times hst ON (h.id = hst.host_id)`+hostMDMSeenTimeJoin+` LEFT JOIN host_dep_assignments hdep ON h.id = hdep.host_id %s %s diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index 58655d33b5..0758d5020f 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -100,6 +100,7 @@ func TestHosts(t *testing.T) { {"SearchWildCards", testSearchHostsWildCards}, {"SearchLimit", testHostsSearchLimit}, {"GenerateStatusStatistics", testHostsGenerateStatusStatistics}, + {"GenerateStatusStatisticsMobileMDMSeenTime", testHostsGenerateStatusStatisticsMobileMDMSeenTime}, {"GenerateStatusStatisticsABMPendingExclusion", testHostsGenerateStatusStatisticsABMPendingExclusion}, {"GenerateStatusStatisticsDEPErrors", testHostsGenerateStatusStatisticsDEPErrors}, {"LowDiskSpaceFilterExcludesSentinel", testHostsLowDiskSpaceFilterExcludesSentinel}, @@ -2978,6 +2979,64 @@ func testHostsGenerateStatusStatistics(t *testing.T, ds *Datastore) { assert.Equal(t, uint(1), *summary.LowDiskSpaceCount) } +// testHostsGenerateStatusStatisticsMobileMDMSeenTime verifies that ios/ipados hosts, which never +// report a host_seen_times entry (no osquery), are not flagged as "missing" when they have recently +// checked in via the Apple MDM protocol (nano_enrollments.last_seen_at). +func testHostsGenerateStatusStatisticsMobileMDMSeenTime(t *testing.T, ds *Datastore) { + ctx := context.Background() + filter := fleet.TeamFilter{User: test.UserAdmin} + now := time.Now() + + // An ios host that enrolled long ago (created_at/detail_updated_at both > 30 days) and never + // checks in via osquery, so it has no host_seen_times row. Without the MDM last_seen_at fallback + // it would be incorrectly counted as missing. + h, err := ds.NewHost(ctx, &fleet.Host{ + Hostname: "ios-device", + UUID: "ios-device-uuid", + HardwareSerial: "ios-serial", + Platform: "ios", + DetailUpdatedAt: now.Add(-40 * 24 * time.Hour), + LabelUpdatedAt: now.Add(-40 * 24 * time.Hour), + PolicyUpdatedAt: now.Add(-40 * 24 * time.Hour), + }) + require.NoError(t, err) + + // Backdate created_at and remove the host_seen_times row to mimic a real ios host. + _, err = ds.writer(ctx).ExecContext(ctx, `UPDATE hosts SET created_at = ? WHERE id = ?`, now.Add(-40*24*time.Hour), h.ID) + require.NoError(t, err) + _, err = ds.writer(ctx).ExecContext(ctx, `DELETE FROM host_seen_times WHERE host_id = ?`, h.ID) + require.NoError(t, err) + + // Recent MDM check-in: device-channel nano enrollment with a fresh last_seen_at. + nanoEnroll(t, ds, h, false) + _, err = ds.writer(ctx).ExecContext(ctx, `UPDATE nano_enrollments SET last_seen_at = ? WHERE id = ?`, now.Add(-1*time.Hour), h.UUID) + require.NoError(t, err) + + missingFilter := fleet.HostListOptions{StatusFilter: fleet.StatusMissing} + + // With a recent MDM last_seen_at, the host must NOT be counted/listed as missing. + summary, err := ds.GenerateHostStatusStatistics(ctx, filter, now, nil, nil) + require.NoError(t, err) + assert.Equal(t, uint(0), summary.Missing30DaysCount, "ios host with recent MDM check-in should not be missing") + + hosts, err := ds.ListHosts(ctx, filter, missingFilter) + require.NoError(t, err) + assert.Empty(t, hosts, "ios host with recent MDM check-in should not appear in missing list") + + // Stale MDM check-in (> 30 days): the host should now be counted/listed as missing. + _, err = ds.writer(ctx).ExecContext(ctx, `UPDATE nano_enrollments SET last_seen_at = ? WHERE id = ?`, now.Add(-40*24*time.Hour), h.UUID) + require.NoError(t, err) + + summary, err = ds.GenerateHostStatusStatistics(ctx, filter, now, nil, nil) + require.NoError(t, err) + assert.Equal(t, uint(1), summary.Missing30DaysCount, "ios host with stale MDM check-in should be missing") + + hosts, err = ds.ListHosts(ctx, filter, missingFilter) + require.NoError(t, err) + require.Len(t, hosts, 1, "ios host with stale MDM check-in should appear in missing list") + assert.Equal(t, h.ID, hosts[0].ID) +} + func testHostsLowDiskSpaceFilterExcludesSentinel(t *testing.T, ds *Datastore) { ctx := context.Background() diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index 9a19e0b346..4bfa8aadb5 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -1271,6 +1271,12 @@ func (ds *Datastore) applyHostLabelFilters(ctx context.Context, filter fleet.Tea // prior to returning, params will be appended in the following order: joinParams, whereParams var whereParams, joinParams []interface{} + // Needed by filterHostsByStatus' missing computation so that ios/ipados hosts fall back to the + // MDM protocol's last_seen_at instead of being flagged missing (see hostEffectiveLastSeenExpr). + if opt.StatusFilter.IsValid() { + query += hostMDMSeenTimeJoin + } + if opt.ListOptions.OrderKey == "display_name" { query += ` JOIN host_display_names hdn ON h.id = hdn.host_id ` } diff --git a/server/datastore/mysql/targets.go b/server/datastore/mysql/targets.go index dfc4865806..3de4c79289 100644 --- a/server/datastore/mysql/targets.go +++ b/server/datastore/mysql/targets.go @@ -20,18 +20,18 @@ func (ds *Datastore) CountHostsInTargets(ctx context.Context, filter fleet.TeamF return fleet.TargetMetrics{}, nil } - queryTargetLogicCondition, queryTargetArgs := targetSQLCondAndArgs(targets) + queryTargetLogicCondition, queryTargetArgs := targetSQLCondAndArgs(targets, "h") // As of Fleet 4.15, mia hosts are also included in the total for offline hosts sql := fmt.Sprintf(` SELECT COUNT(*) total, - COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) <= ? THEN 1 ELSE 0 END), 0) mia, + COALESCE(SUM(CASE WHEN DATE_ADD(`+hostEffectiveLastSeenExpr+`, INTERVAL 30 DAY) <= ? THEN 1 ELSE 0 END), 0) mia, COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) <= ? THEN 1 ELSE 0 END), 0) offline, COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) > ? THEN 1 ELSE 0 END), 0) online, - COALESCE(SUM(CASE WHEN DATE_ADD(created_at, INTERVAL 1 DAY) >= ? THEN 1 ELSE 0 END), 0) new + COALESCE(SUM(CASE WHEN DATE_ADD(h.created_at, INTERVAL 1 DAY) >= ? THEN 1 ELSE 0 END), 0) new FROM hosts h - LEFT JOIN host_seen_times hst ON (h.id=hst.host_id) + LEFT JOIN host_seen_times hst ON (h.id=hst.host_id)`+hostMDMSeenTimeJoin+` WHERE %s AND %s`, fleet.OnlineIntervalBuffer, fleet.OnlineIntervalBuffer, queryTargetLogicCondition, ds.whereFilterHostsByTeams(filter, "h"), @@ -53,14 +53,14 @@ func (ds *Datastore) CountHostsInTargets(ctx context.Context, filter fleet.TeamF // targetSQLCondAndArgs returns the SQL condition and the arguments for matching whether // a host ID is a target of a live query. -func targetSQLCondAndArgs(targets fleet.HostTargets) (sql string, args []interface{}) { +func targetSQLCondAndArgs(targets fleet.HostTargets, hostKey string) (sql string, args []any) { const queryTargetLogicCondition = `( /* The host was selected explicitly. */ - id IN (? /* queryHostIDs */) + %[1]s.id IN (? /* queryHostIDs */) OR ( /* 'All hosts' builtin label was selected. */ - id IN (SELECT DISTINCT host_id FROM label_membership WHERE label_id = (SELECT id from labels WHERE name = 'All Hosts') AND label_id IN (? /* queryLabelIDs */)) + %[1]s.id IN (SELECT DISTINCT host_id FROM label_membership WHERE label_id = (SELECT id from labels WHERE name = 'All Hosts') AND label_id IN (? /* queryLabelIDs */)) ) OR ( @@ -72,7 +72,7 @@ func targetSQLCondAndArgs(targets fleet.HostTargets) (sql string, args []interfa ( SELECT NOT EXISTS (SELECT id FROM labels WHERE label_type <> 1 AND id IN (? /* queryLabelIDs */)) OR - (id IN (SELECT DISTINCT host_id FROM label_membership lm JOIN labels l ON lm.label_id = l.id WHERE l.label_type <> 1 AND lm.label_id IN (? /* queryLabelIDs */))) + (%[1]s.id IN (SELECT DISTINCT host_id FROM label_membership lm JOIN labels l ON lm.label_id = l.id WHERE l.label_type <> 1 AND lm.label_id IN (? /* queryLabelIDs */))) ) AND /* A builtin label filter was not specified OR if it was specified then the host must be @@ -80,7 +80,7 @@ func targetSQLCondAndArgs(targets fleet.HostTargets) (sql string, args []interfa ( SELECT NOT EXISTS (SELECT id FROM labels WHERE label_type = 1 AND id IN (? /* queryLabelIDs */)) OR - (id IN (SELECT DISTINCT host_id FROM label_membership lm JOIN labels l ON lm.label_id = l.id WHERE l.label_type = 1 AND lm.label_id IN (? /* queryLabelIDs */))) + (%[1]s.id IN (SELECT DISTINCT host_id FROM label_membership lm JOIN labels l ON lm.label_id = l.id WHERE l.label_type = 1 AND lm.label_id IN (? /* queryLabelIDs */))) ) AND /* A team filter was not specified OR if it was specified then the host must be a @@ -114,7 +114,7 @@ func targetSQLCondAndArgs(targets fleet.HostTargets) (sql string, args []interfa labelsSpecified := len(queryLabelIDs) > 1 teamsSpecified := len(queryTeamIDs) > 1 || extraTeamIDCondition != "" - return fmt.Sprintf(queryTargetLogicCondition, extraTeamIDCondition), []interface{}{ + return fmt.Sprintf(queryTargetLogicCondition, hostKey, extraTeamIDCondition), []any{ queryHostIDs, queryLabelIDs, labelsSpecified, teamsSpecified, @@ -130,7 +130,7 @@ func (ds *Datastore) HostIDsInTargets(ctx context.Context, filter fleet.TeamFilt return []uint{}, nil } - queryTargetLogicCondition, queryTargetArgs := targetSQLCondAndArgs(targets) + queryTargetLogicCondition, queryTargetArgs := targetSQLCondAndArgs(targets, "hosts") sql := fmt.Sprintf(` SELECT DISTINCT id diff --git a/server/fleet/hosts.go b/server/fleet/hosts.go index ed433b0ce6..3e5a5df4fc 100644 --- a/server/fleet/hosts.go +++ b/server/fleet/hosts.go @@ -1101,9 +1101,8 @@ type HostSummaryPlatform struct { // Status calculates the online status of the host func (h *Host) Status(now time.Time) HostStatus { // The logic in this function should remain synchronized with - // GenerateHostStatusStatistics and CountHostsInTargets + // GenerateHostStatusStatistics and CountHostsInTargets - it can't stay in sync for MDM join, since that attribute is not available. // NOTE: As of Fleet 4.15 StatusMIA is deprecated and will be removed in Fleet 5.0 - onlineInterval := h.ConfigTLSRefresh if h.DistributedInterval < h.ConfigTLSRefresh { onlineInterval = h.DistributedInterval