Join MDM for missing status for non osquery devices (#47672)
<!-- Add the related story/sub-task/bug number, like Resolves #123, or remove if NA --> **Related issue:** Resolves #46243 # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files) for more information. - [x] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements), JS inline code is prevented especially for url redirects, and untrusted data interpolated into shell scripts/commands is validated against shell metacharacters. - [x] Timeouts are implemented and retries are limited to avoid infinite loops - [x] If paths of existing endpoints are modified without backwards compatibility, checked the frontend/CLI for any necessary changes ## Testing - [x] Added/updated automated tests - [x] QA'd all new/changed functionality manually <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **Bug Fixes** * Fixed incorrect reporting of iOS, iPadOS, and Android hosts as missing (including MIA and missing-over-30-days counts). * Host status filtering and dashboard cards now correctly fall back to Apple MDM activity when OSQuery “last seen” data is unavailable, preventing recently active devices from being flagged. * **Tests** * Added coverage to verify iOS/iPadOS hosts aren’t treated as missing when Apple MDM “last seen” is recent, and are flagged once it becomes stale. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Magnus Jensen
@@ -0,0 +1 @@
|
||||
* Fixed an issue where Missing hosts filter and dashboard card incorrectly reported iOS, iPadOS, and Android hosts.
|
||||
@@ -1431,6 +1431,11 @@ func (ds *Datastore) applyHostFilters(
|
||||
batchScriptExecutionJoin, batchScriptExecutionFilter, whereParams = ds.getBatchExecutionFilters(whereParams, opt)
|
||||
}
|
||||
|
||||
hostMDMSeenJoin := ""
|
||||
if opt.StatusFilter.IsValid() {
|
||||
hostMDMSeenJoin = hostMDMSeenTimeJoin
|
||||
}
|
||||
|
||||
var depStatusFilter string
|
||||
wantFailedDEP := ptr.ValOrZero(opt.DEPProfileErrorFilter)
|
||||
wantDepResp := string(ptr.ValOrZero(opt.DEPAssignProfileResponseFilter))
|
||||
@@ -1462,6 +1467,7 @@ func (ds *Datastore) applyHostFilters(
|
||||
%s
|
||||
%s
|
||||
%s
|
||||
%s
|
||||
%s
|
||||
WHERE TRUE AND %s AND %s AND %s AND %s AND %s %s
|
||||
`,
|
||||
@@ -1481,6 +1487,7 @@ func (ds *Datastore) applyHostFilters(
|
||||
mdmRecoveryLockStatusJoin,
|
||||
mdmAndroidProfilesStatusJoin,
|
||||
batchScriptExecutionJoin,
|
||||
hostMDMSeenJoin,
|
||||
|
||||
// Conditions
|
||||
ds.whereFilterHostsByTeams(filter, "h"),
|
||||
@@ -1652,6 +1659,19 @@ func filterHostsByPolicy(sql string, opt fleet.HostListOptions, params []interfa
|
||||
return sql, params
|
||||
}
|
||||
|
||||
// hostMDMSeenTimeJoin joins on nano enrollment so that the effective last-seen
|
||||
// time can fall back to the Apple MDM protocol's last_seen_at
|
||||
// for hosts that never check in via osquery (ios/ipados).
|
||||
// It uses a dedicated alias (nes) to avoid colliding with the connected-to-Fleet join (ne)
|
||||
const hostMDMSeenTimeJoin = `
|
||||
LEFT JOIN nano_enrollments nes ON nes.id = h.uuid AND nes.type IN ('Device', 'User Enrollment (Device)')`
|
||||
|
||||
// hostEffectiveLastSeenExpr is the effective "last seen" time for a host: the greatest of the osquery
|
||||
// seen_time and the MDM last_seen_at, then detail_updated_at (treating the Never sentinel as null),
|
||||
// then created_at.
|
||||
// Requires hostMDMSeenTimeJoin (alias nes) and the host_seen_times join (alias hst) to be present.
|
||||
const hostEffectiveLastSeenExpr = `COALESCE(GREATEST(COALESCE(hst.seen_time, nes.last_seen_at), COALESCE(nes.last_seen_at, hst.seen_time)), NULLIF(h.detail_updated_at, '` + server.NeverTimestamp + `'), h.created_at)`
|
||||
|
||||
func filterHostsByStatus(now time.Time, sql string, opt fleet.HostListOptions, params []interface{}) (string, []interface{}) {
|
||||
switch opt.StatusFilter {
|
||||
case fleet.StatusNew:
|
||||
@@ -1664,7 +1684,8 @@ func filterHostsByStatus(now time.Time, sql string, opt fleet.HostListOptions, p
|
||||
sql += fmt.Sprintf("AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(h.distributed_interval, h.config_tls_refresh) + %d SECOND) <= ?", fleet.OnlineIntervalBuffer)
|
||||
params = append(params, now)
|
||||
case fleet.StatusMIA, fleet.StatusMissing:
|
||||
sql += "AND DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) <= ? AND (hmdm.enrollment_status IS NULL OR hmdm.enrollment_status != 'Pending')"
|
||||
// This must stay in sync with the missing_30_days_count computation in GenerateHostStatusStatistics.
|
||||
sql += "AND DATE_ADD(" + hostEffectiveLastSeenExpr + ", INTERVAL 30 DAY) <= ? AND (hmdm.enrollment_status IS NULL OR hmdm.enrollment_status != 'Pending')"
|
||||
params = append(params, now)
|
||||
}
|
||||
return sql, params
|
||||
@@ -2140,15 +2161,15 @@ func (ds *Datastore) GenerateHostStatusStatistics(ctx context.Context, filter fl
|
||||
sqlStatement := fmt.Sprintf(`
|
||||
SELECT
|
||||
COUNT(*) total,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) <= ? AND (hmdm.enrollment_status IS NULL OR hmdm.enrollment_status != 'Pending') THEN 1 ELSE 0 END), 0) mia,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) <= ? AND (hmdm.enrollment_status IS NULL OR hmdm.enrollment_status != 'Pending') THEN 1 ELSE 0 END), 0) missing_30_days_count,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(`+hostEffectiveLastSeenExpr+`, INTERVAL 30 DAY) <= ? AND (hmdm.enrollment_status IS NULL OR hmdm.enrollment_status != 'Pending') THEN 1 ELSE 0 END), 0) mia,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(`+hostEffectiveLastSeenExpr+`, INTERVAL 30 DAY) <= ? AND (hmdm.enrollment_status IS NULL OR hmdm.enrollment_status != 'Pending') THEN 1 ELSE 0 END), 0) missing_30_days_count,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) <= ? THEN 1 ELSE 0 END), 0) offline,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) > ? THEN 1 ELSE 0 END), 0) online,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(h.created_at, INTERVAL 1 DAY) >= ? THEN 1 ELSE 0 END), 0) new,
|
||||
COALESCE(SUM(CASE WHEN hdep.assign_profile_response IN (%s, %s) THEN 1 ELSE 0 END), 0) dep_assign_error_count,
|
||||
%s
|
||||
FROM hosts h
|
||||
LEFT JOIN host_seen_times hst ON (h.id = hst.host_id)
|
||||
LEFT JOIN host_seen_times hst ON (h.id = hst.host_id)`+hostMDMSeenTimeJoin+`
|
||||
LEFT JOIN host_dep_assignments hdep ON h.id = hdep.host_id
|
||||
%s
|
||||
%s
|
||||
|
||||
@@ -100,6 +100,7 @@ func TestHosts(t *testing.T) {
|
||||
{"SearchWildCards", testSearchHostsWildCards},
|
||||
{"SearchLimit", testHostsSearchLimit},
|
||||
{"GenerateStatusStatistics", testHostsGenerateStatusStatistics},
|
||||
{"GenerateStatusStatisticsMobileMDMSeenTime", testHostsGenerateStatusStatisticsMobileMDMSeenTime},
|
||||
{"GenerateStatusStatisticsABMPendingExclusion", testHostsGenerateStatusStatisticsABMPendingExclusion},
|
||||
{"GenerateStatusStatisticsDEPErrors", testHostsGenerateStatusStatisticsDEPErrors},
|
||||
{"LowDiskSpaceFilterExcludesSentinel", testHostsLowDiskSpaceFilterExcludesSentinel},
|
||||
@@ -2978,6 +2979,64 @@ func testHostsGenerateStatusStatistics(t *testing.T, ds *Datastore) {
|
||||
assert.Equal(t, uint(1), *summary.LowDiskSpaceCount)
|
||||
}
|
||||
|
||||
// testHostsGenerateStatusStatisticsMobileMDMSeenTime verifies that ios/ipados hosts, which never
|
||||
// report a host_seen_times entry (no osquery), are not flagged as "missing" when they have recently
|
||||
// checked in via the Apple MDM protocol (nano_enrollments.last_seen_at).
|
||||
func testHostsGenerateStatusStatisticsMobileMDMSeenTime(t *testing.T, ds *Datastore) {
|
||||
ctx := context.Background()
|
||||
filter := fleet.TeamFilter{User: test.UserAdmin}
|
||||
now := time.Now()
|
||||
|
||||
// An ios host that enrolled long ago (created_at/detail_updated_at both > 30 days) and never
|
||||
// checks in via osquery, so it has no host_seen_times row. Without the MDM last_seen_at fallback
|
||||
// it would be incorrectly counted as missing.
|
||||
h, err := ds.NewHost(ctx, &fleet.Host{
|
||||
Hostname: "ios-device",
|
||||
UUID: "ios-device-uuid",
|
||||
HardwareSerial: "ios-serial",
|
||||
Platform: "ios",
|
||||
DetailUpdatedAt: now.Add(-40 * 24 * time.Hour),
|
||||
LabelUpdatedAt: now.Add(-40 * 24 * time.Hour),
|
||||
PolicyUpdatedAt: now.Add(-40 * 24 * time.Hour),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Backdate created_at and remove the host_seen_times row to mimic a real ios host.
|
||||
_, err = ds.writer(ctx).ExecContext(ctx, `UPDATE hosts SET created_at = ? WHERE id = ?`, now.Add(-40*24*time.Hour), h.ID)
|
||||
require.NoError(t, err)
|
||||
_, err = ds.writer(ctx).ExecContext(ctx, `DELETE FROM host_seen_times WHERE host_id = ?`, h.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Recent MDM check-in: device-channel nano enrollment with a fresh last_seen_at.
|
||||
nanoEnroll(t, ds, h, false)
|
||||
_, err = ds.writer(ctx).ExecContext(ctx, `UPDATE nano_enrollments SET last_seen_at = ? WHERE id = ?`, now.Add(-1*time.Hour), h.UUID)
|
||||
require.NoError(t, err)
|
||||
|
||||
missingFilter := fleet.HostListOptions{StatusFilter: fleet.StatusMissing}
|
||||
|
||||
// With a recent MDM last_seen_at, the host must NOT be counted/listed as missing.
|
||||
summary, err := ds.GenerateHostStatusStatistics(ctx, filter, now, nil, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint(0), summary.Missing30DaysCount, "ios host with recent MDM check-in should not be missing")
|
||||
|
||||
hosts, err := ds.ListHosts(ctx, filter, missingFilter)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, hosts, "ios host with recent MDM check-in should not appear in missing list")
|
||||
|
||||
// Stale MDM check-in (> 30 days): the host should now be counted/listed as missing.
|
||||
_, err = ds.writer(ctx).ExecContext(ctx, `UPDATE nano_enrollments SET last_seen_at = ? WHERE id = ?`, now.Add(-40*24*time.Hour), h.UUID)
|
||||
require.NoError(t, err)
|
||||
|
||||
summary, err = ds.GenerateHostStatusStatistics(ctx, filter, now, nil, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint(1), summary.Missing30DaysCount, "ios host with stale MDM check-in should be missing")
|
||||
|
||||
hosts, err = ds.ListHosts(ctx, filter, missingFilter)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, hosts, 1, "ios host with stale MDM check-in should appear in missing list")
|
||||
assert.Equal(t, h.ID, hosts[0].ID)
|
||||
}
|
||||
|
||||
func testHostsLowDiskSpaceFilterExcludesSentinel(t *testing.T, ds *Datastore) {
|
||||
ctx := context.Background()
|
||||
|
||||
|
||||
@@ -1271,6 +1271,12 @@ func (ds *Datastore) applyHostLabelFilters(ctx context.Context, filter fleet.Tea
|
||||
// prior to returning, params will be appended in the following order: joinParams, whereParams
|
||||
var whereParams, joinParams []interface{}
|
||||
|
||||
// Needed by filterHostsByStatus' missing computation so that ios/ipados hosts fall back to the
|
||||
// MDM protocol's last_seen_at instead of being flagged missing (see hostEffectiveLastSeenExpr).
|
||||
if opt.StatusFilter.IsValid() {
|
||||
query += hostMDMSeenTimeJoin
|
||||
}
|
||||
|
||||
if opt.ListOptions.OrderKey == "display_name" {
|
||||
query += ` JOIN host_display_names hdn ON h.id = hdn.host_id `
|
||||
}
|
||||
|
||||
@@ -20,18 +20,18 @@ func (ds *Datastore) CountHostsInTargets(ctx context.Context, filter fleet.TeamF
|
||||
return fleet.TargetMetrics{}, nil
|
||||
}
|
||||
|
||||
queryTargetLogicCondition, queryTargetArgs := targetSQLCondAndArgs(targets)
|
||||
queryTargetLogicCondition, queryTargetArgs := targetSQLCondAndArgs(targets, "h")
|
||||
|
||||
// As of Fleet 4.15, mia hosts are also included in the total for offline hosts
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT
|
||||
COUNT(*) total,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL 30 DAY) <= ? THEN 1 ELSE 0 END), 0) mia,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(`+hostEffectiveLastSeenExpr+`, INTERVAL 30 DAY) <= ? THEN 1 ELSE 0 END), 0) mia,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) <= ? THEN 1 ELSE 0 END), 0) offline,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(COALESCE(hst.seen_time, h.created_at), INTERVAL LEAST(distributed_interval, config_tls_refresh) + %d SECOND) > ? THEN 1 ELSE 0 END), 0) online,
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(created_at, INTERVAL 1 DAY) >= ? THEN 1 ELSE 0 END), 0) new
|
||||
COALESCE(SUM(CASE WHEN DATE_ADD(h.created_at, INTERVAL 1 DAY) >= ? THEN 1 ELSE 0 END), 0) new
|
||||
FROM hosts h
|
||||
LEFT JOIN host_seen_times hst ON (h.id=hst.host_id)
|
||||
LEFT JOIN host_seen_times hst ON (h.id=hst.host_id)`+hostMDMSeenTimeJoin+`
|
||||
WHERE %s AND %s`,
|
||||
fleet.OnlineIntervalBuffer, fleet.OnlineIntervalBuffer,
|
||||
queryTargetLogicCondition, ds.whereFilterHostsByTeams(filter, "h"),
|
||||
@@ -53,14 +53,14 @@ func (ds *Datastore) CountHostsInTargets(ctx context.Context, filter fleet.TeamF
|
||||
|
||||
// targetSQLCondAndArgs returns the SQL condition and the arguments for matching whether
|
||||
// a host ID is a target of a live query.
|
||||
func targetSQLCondAndArgs(targets fleet.HostTargets) (sql string, args []interface{}) {
|
||||
func targetSQLCondAndArgs(targets fleet.HostTargets, hostKey string) (sql string, args []any) {
|
||||
const queryTargetLogicCondition = `(
|
||||
/* The host was selected explicitly. */
|
||||
id IN (? /* queryHostIDs */)
|
||||
%[1]s.id IN (? /* queryHostIDs */)
|
||||
OR
|
||||
(
|
||||
/* 'All hosts' builtin label was selected. */
|
||||
id IN (SELECT DISTINCT host_id FROM label_membership WHERE label_id = (SELECT id from labels WHERE name = 'All Hosts') AND label_id IN (? /* queryLabelIDs */))
|
||||
%[1]s.id IN (SELECT DISTINCT host_id FROM label_membership WHERE label_id = (SELECT id from labels WHERE name = 'All Hosts') AND label_id IN (? /* queryLabelIDs */))
|
||||
)
|
||||
OR
|
||||
(
|
||||
@@ -72,7 +72,7 @@ func targetSQLCondAndArgs(targets fleet.HostTargets) (sql string, args []interfa
|
||||
(
|
||||
SELECT NOT EXISTS (SELECT id FROM labels WHERE label_type <> 1 AND id IN (? /* queryLabelIDs */))
|
||||
OR
|
||||
(id IN (SELECT DISTINCT host_id FROM label_membership lm JOIN labels l ON lm.label_id = l.id WHERE l.label_type <> 1 AND lm.label_id IN (? /* queryLabelIDs */)))
|
||||
(%[1]s.id IN (SELECT DISTINCT host_id FROM label_membership lm JOIN labels l ON lm.label_id = l.id WHERE l.label_type <> 1 AND lm.label_id IN (? /* queryLabelIDs */)))
|
||||
)
|
||||
AND
|
||||
/* A builtin label filter was not specified OR if it was specified then the host must be
|
||||
@@ -80,7 +80,7 @@ func targetSQLCondAndArgs(targets fleet.HostTargets) (sql string, args []interfa
|
||||
(
|
||||
SELECT NOT EXISTS (SELECT id FROM labels WHERE label_type = 1 AND id IN (? /* queryLabelIDs */))
|
||||
OR
|
||||
(id IN (SELECT DISTINCT host_id FROM label_membership lm JOIN labels l ON lm.label_id = l.id WHERE l.label_type = 1 AND lm.label_id IN (? /* queryLabelIDs */)))
|
||||
(%[1]s.id IN (SELECT DISTINCT host_id FROM label_membership lm JOIN labels l ON lm.label_id = l.id WHERE l.label_type = 1 AND lm.label_id IN (? /* queryLabelIDs */)))
|
||||
)
|
||||
AND
|
||||
/* A team filter was not specified OR if it was specified then the host must be a
|
||||
@@ -114,7 +114,7 @@ func targetSQLCondAndArgs(targets fleet.HostTargets) (sql string, args []interfa
|
||||
labelsSpecified := len(queryLabelIDs) > 1
|
||||
teamsSpecified := len(queryTeamIDs) > 1 || extraTeamIDCondition != ""
|
||||
|
||||
return fmt.Sprintf(queryTargetLogicCondition, extraTeamIDCondition), []interface{}{
|
||||
return fmt.Sprintf(queryTargetLogicCondition, hostKey, extraTeamIDCondition), []any{
|
||||
queryHostIDs,
|
||||
queryLabelIDs,
|
||||
labelsSpecified, teamsSpecified,
|
||||
@@ -130,7 +130,7 @@ func (ds *Datastore) HostIDsInTargets(ctx context.Context, filter fleet.TeamFilt
|
||||
return []uint{}, nil
|
||||
}
|
||||
|
||||
queryTargetLogicCondition, queryTargetArgs := targetSQLCondAndArgs(targets)
|
||||
queryTargetLogicCondition, queryTargetArgs := targetSQLCondAndArgs(targets, "hosts")
|
||||
|
||||
sql := fmt.Sprintf(`
|
||||
SELECT DISTINCT id
|
||||
|
||||
@@ -1101,9 +1101,8 @@ type HostSummaryPlatform struct {
|
||||
// Status calculates the online status of the host
|
||||
func (h *Host) Status(now time.Time) HostStatus {
|
||||
// The logic in this function should remain synchronized with
|
||||
// GenerateHostStatusStatistics and CountHostsInTargets
|
||||
// GenerateHostStatusStatistics and CountHostsInTargets - it can't stay in sync for MDM join, since that attribute is not available.
|
||||
// NOTE: As of Fleet 4.15 StatusMIA is deprecated and will be removed in Fleet 5.0
|
||||
|
||||
onlineInterval := h.ConfigTLSRefresh
|
||||
if h.DistributedInterval < h.ConfigTLSRefresh {
|
||||
onlineInterval = h.DistributedInterval
|
||||
|
||||
Reference in New Issue
Block a user