From 3f9eccc7f816091f94f5023aaeb55e4b3ed4be82 Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 17 May 2023 15:52:45 -0400 Subject: [PATCH] Refetch host mdm enrollment status until unenrolled (#11740) --- changes/issue-11695-webhook-migrate-mdm | 1 + ee/server/service/devices.go | 22 +++++- server/datastore/mysql/hosts.go | 22 +++++- server/datastore/mysql/hosts_test.go | 11 +++ server/datastore/mysql/labels.go | 14 ++-- ...7_AddRefetchCriticalQueriesUntilToHosts.go | 22 ++++++ ...RefetchCriticalQueriesUntilToHosts_test.go | 79 +++++++++++++++++++ server/datastore/mysql/schema.sql | 5 +- server/fleet/hosts.go | 16 ++++ server/service/integration_mdm_test.go | 43 +++++++++- server/service/osquery.go | 25 +++++- server/service/osquery_test.go | 37 ++++++++- server/service/osquery_utils/queries.go | 9 ++- 13 files changed, 283 insertions(+), 23 deletions(-) create mode 100644 server/datastore/mysql/migrations/tables/20230517152807_AddRefetchCriticalQueriesUntilToHosts.go create mode 100644 server/datastore/mysql/migrations/tables/20230517152807_AddRefetchCriticalQueriesUntilToHosts_test.go diff --git a/changes/issue-11695-webhook-migrate-mdm b/changes/issue-11695-webhook-migrate-mdm index 8d8b16333a..6c49b4bfe8 100644 --- a/changes/issue-11695-webhook-migrate-mdm +++ b/changes/issue-11695-webhook-migrate-mdm @@ -1 +1,2 @@ - Added device-authenticated endpoint to signal the Fleet server to send a webbook request with the device UUID and serial number to the webhook URL configured for MDM migration. +- Added mechanism to refetch MDM enrollment status of a host pending unenrollment (due to a migration to Fleet) at a high interval. diff --git a/ee/server/service/devices.go b/ee/server/service/devices.go index fc060c5c59..a68c1ad990 100644 --- a/ee/server/service/devices.go +++ b/ee/server/service/devices.go @@ -18,6 +18,12 @@ func (svc *Service) RequestEncryptionKeyRotation(ctx context.Context, hostID uin return svc.ds.SetDiskEncryptionResetStatus(ctx, hostID, true) } +const refetchMDMUnenrollCriticalQueryDuration = 3 * time.Minute + +// TriggerMigrateMDMDevice triggers the webhook associated with the MDM +// migration to Fleet configuration. It is located in the ee package instead of +// the server/webhooks one because it is a Fleet Premium only feature and for +// licensing reasons this needs to live under this package. func (svc *Service) TriggerMigrateMDMDevice(ctx context.Context, host *fleet.Host) error { ac, err := svc.ds.AppConfig(ctx) if err != nil { @@ -35,9 +41,12 @@ func (svc *Service) TriggerMigrateMDMDevice(ctx context.Context, host *fleet.Hos bre.InternalErr = ctxerr.New(ctx, "macOS migration webhook URL not configured") case !host.IsOsqueryEnrolled(), !host.MDMInfo.IsDEPCapable(), !host.MDMInfo.IsEnrolledInThirdPartyMDM(): bre.InternalErr = ctxerr.New(ctx, "host not eligible for macOS migration") + case host.RefetchCriticalQueriesUntil != nil && host.RefetchCriticalQueriesUntil.After(svc.clock.Now()): + // the webhook has already been triggered successfully recently (within the + // refetch critical queries delay), so do as if it did send it successfully + // but do not re-send. + return nil } - // TODO: add case to check if webhok has already been sent (if host refetchUntil is not zero?) - if bre.InternalErr != nil { return &bre } @@ -52,6 +61,15 @@ func (svc *Service) TriggerMigrateMDMDevice(ctx context.Context, host *fleet.Hos return ctxerr.Wrap(ctx, err, "posting macOS migration webhook") } + // if the webhook was successfully triggered, we update the host to + // constantly run the query to check if it has been unenrolled from its + // existing third-party MDM. + refetchUntil := svc.clock.Now().Add(refetchMDMUnenrollCriticalQueryDuration) + host.RefetchCriticalQueriesUntil = &refetchUntil + if err := svc.ds.UpdateHost(ctx, host); err != nil { + return ctxerr.Wrap(ctx, err, "save host with refetch critical queries timestamp") + } + return nil } diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index a50d1ae388..fca7bd1d51 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -61,9 +61,10 @@ func (ds *Datastore) NewHost(ctx context.Context, host *fleet.Host) (*fleet.Host logger_tls_period, config_tls_refresh, refetch_requested, - hardware_serial + hardware_serial, + refetch_critical_queries_until ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ` result, err := tx.ExecContext( ctx, @@ -87,6 +88,7 @@ func (ds *Datastore) NewHost(ctx context.Context, host *fleet.Host) (*fleet.Host host.ConfigTLSRefresh, host.RefetchRequested, host.HardwareSerial, + host.RefetchCriticalQueriesUntil, ) if err != nil { return ctxerr.Wrap(ctx, err, "new host") @@ -426,6 +428,7 @@ SELECT h.label_updated_at, h.last_enrolled_at, h.refetch_requested, + h.refetch_critical_queries_until, h.team_id, h.policy_updated_at, h.public_ip, @@ -621,6 +624,7 @@ func (ds *Datastore) ListHosts(ctx context.Context, filter fleet.TeamFilter, opt h.label_updated_at, h.last_enrolled_at, h.refetch_requested, + h.refetch_critical_queries_until, h.team_id, h.policy_updated_at, h.public_ip, @@ -1402,6 +1406,7 @@ func (ds *Datastore) EnrollHost(ctx context.Context, isMDMEnabled bool, osqueryH h.label_updated_at, h.last_enrolled_at, h.refetch_requested, + h.refetch_critical_queries_until, h.team_id, h.policy_updated_at, h.public_ip, @@ -1483,6 +1488,7 @@ func (ds *Datastore) LoadHostByNodeKey(ctx context.Context, nodeKey string) (*fl h.label_updated_at, h.last_enrolled_at, h.refetch_requested, + h.refetch_critical_queries_until, h.team_id, h.policy_updated_at, h.public_ip, @@ -1557,6 +1563,7 @@ func (ds *Datastore) LoadHostByOrbitNodeKey(ctx context.Context, nodeKey string) h.label_updated_at, h.last_enrolled_at, h.refetch_requested, + h.refetch_critical_queries_until, h.team_id, h.policy_updated_at, h.public_ip, @@ -1650,6 +1657,7 @@ func (ds *Datastore) LoadHostByDeviceAuthToken(ctx context.Context, authToken st h.label_updated_at, h.last_enrolled_at, h.refetch_requested, + h.refetch_critical_queries_until, h.team_id, h.policy_updated_at, h.public_ip, @@ -1795,6 +1803,7 @@ func (ds *Datastore) SearchHosts(ctx context.Context, filter fleet.TeamFilter, m h.label_updated_at, h.last_enrolled_at, h.refetch_requested, + h.refetch_critical_queries_until, h.team_id, h.policy_updated_at, h.public_ip, @@ -1890,7 +1899,8 @@ SELECT label_updated_at, last_enrolled_at, policy_updated_at, - refetch_requested + refetch_requested, + refetch_critical_queries_until FROM hosts WHERE uuid IN (?) AND %s `, ds.whereFilterHostsByTeams(filter, "hosts"), @@ -1947,6 +1957,7 @@ func (ds *Datastore) HostByIdentifier(ctx context.Context, identifier string) (* h.label_updated_at, h.last_enrolled_at, h.refetch_requested, + h.refetch_critical_queries_until, h.team_id, h.policy_updated_at, h.public_ip, @@ -3350,6 +3361,7 @@ func (ds *Datastore) HostLite(ctx context.Context, id uint) (*fleet.Host, error) "last_enrolled_at", "policy_updated_at", "refetch_requested", + "refetch_critical_queries_until", ).Where(goqu.I("id").Eq(id)).ToSQL() if err != nil { return nil, ctxerr.Wrap(ctx, err, "sql build") @@ -3432,7 +3444,8 @@ func (ds *Datastore) UpdateHost(ctx context.Context, host *fleet.Host) error { primary_mac = ?, public_ip = ?, refetch_requested = ?, - orbit_node_key = ? + orbit_node_key = ?, + refetch_critical_queries_until = ? WHERE id = ? ` _, err := ds.writer.ExecContext(ctx, sqlStatement, @@ -3469,6 +3482,7 @@ func (ds *Datastore) UpdateHost(ctx context.Context, host *fleet.Host) error { host.PublicIP, host.RefetchRequested, host.OrbitNodeKey, + host.RefetchCriticalQueriesUntil, host.ID, ) if err != nil { diff --git a/server/datastore/mysql/hosts_test.go b/server/datastore/mysql/hosts_test.go index 61a2610b62..b6d63ee16b 100644 --- a/server/datastore/mysql/hosts_test.go +++ b/server/datastore/mysql/hosts_test.go @@ -182,6 +182,10 @@ func testUpdateHost(t *testing.T, ds *Datastore, updateHostFunc func(context.Con err = updateHostFunc(context.Background(), host) require.NoError(t, err) + host.RefetchCriticalQueriesUntil = ptr.Time(time.Now().UTC().Add(time.Hour)) + err = updateHostFunc(context.Background(), host) + require.NoError(t, err) + host, err = ds.Host(context.Background(), host.ID) require.NoError(t, err) @@ -189,6 +193,8 @@ func testUpdateHost(t *testing.T, ds *Datastore, updateHostFunc func(context.Con assert.Equal(t, "192.168.1.1", host.PrimaryIP) assert.Equal(t, "30-65-EC-6F-C4-58", host.PrimaryMac) assert.Equal(t, policyUpdatedAt.UTC(), host.PolicyUpdatedAt) + assert.NotNil(t, host.RefetchCriticalQueriesUntil) + assert.True(t, time.Now().Before(*host.RefetchCriticalQueriesUntil)) additionalJSON := json.RawMessage(`{"foobar": "bim"}`) err = ds.SaveHostAdditional(context.Background(), host.ID, &additionalJSON) @@ -203,9 +209,14 @@ func testUpdateHost(t *testing.T, ds *Datastore, updateHostFunc func(context.Con err = updateHostFunc(context.Background(), host) require.NoError(t, err) + host.RefetchCriticalQueriesUntil = nil + err = updateHostFunc(context.Background(), host) + require.NoError(t, err) + host, err = ds.Host(context.Background(), host.ID) require.NoError(t, err) require.NotNil(t, host) + require.Nil(t, host.RefetchCriticalQueriesUntil) p, err := ds.NewPack(context.Background(), &fleet.Pack{ Name: t.Name(), diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index 4364e9699c..d678819cf1 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -497,24 +497,25 @@ func (ds *Datastore) ListHostsInLabel(ctx context.Context, filter fleet.TeamFilt h.label_updated_at, h.last_enrolled_at, h.refetch_requested, + h.refetch_critical_queries_until, h.team_id, h.policy_updated_at, h.public_ip, COALESCE(hd.gigs_disk_space_available, 0) as gigs_disk_space_available, COALESCE(hd.percent_disk_space_available, 0) as percent_disk_space_available, COALESCE(hst.seen_time, h.created_at) as seen_time, - COALESCE(hu.software_updated_at, h.created_at) AS software_updated_at, + COALESCE(hu.software_updated_at, h.created_at) AS software_updated_at, (SELECT name FROM teams t WHERE t.id = h.team_id) AS team_name %s - %s + %s FROM label_membership lm JOIN hosts h ON (lm.host_id = h.id) LEFT JOIN host_seen_times hst ON (h.id=hst.host_id) - LEFT JOIN host_updates hu ON (h.id = hu.host_id) + LEFT JOIN host_updates hu ON (h.id = hu.host_id) LEFT JOIN host_disks hd ON (h.id=hd.host_id) %s - %s - ` + %s +` failingPoliciesSelect := `, COALESCE(failing_policies.count, 0) AS failing_policies_count, COALESCE(failing_policies.count, 0) AS total_issues_count @@ -631,6 +632,7 @@ func (ds *Datastore) ListUniqueHostsInLabels(ctx context.Context, filter fleet.T h.label_updated_at, h.last_enrolled_at, h.refetch_requested, + h.refetch_critical_queries_until, h.team_id, h.policy_updated_at, h.public_ip, @@ -641,7 +643,7 @@ func (ds *Datastore) ListUniqueHostsInLabels(ctx context.Context, filter fleet.T JOIN hosts h ON lm.host_id = h.id LEFT JOIN host_disks hd ON hd.host_id = h.id WHERE lm.label_id IN (?) AND %s - `, ds.whereFilterHostsByTeams(filter, "h"), +`, ds.whereFilterHostsByTeams(filter, "h"), ) query, args, err := sqlx.In(sqlStatement, labels) diff --git a/server/datastore/mysql/migrations/tables/20230517152807_AddRefetchCriticalQueriesUntilToHosts.go b/server/datastore/mysql/migrations/tables/20230517152807_AddRefetchCriticalQueriesUntilToHosts.go new file mode 100644 index 0000000000..f50290b613 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20230517152807_AddRefetchCriticalQueriesUntilToHosts.go @@ -0,0 +1,22 @@ +package tables + +import ( + "database/sql" + + "github.com/pkg/errors" +) + +func init() { + MigrationClient.AddMigration(Up_20230517152807, Down_20230517152807) +} + +func Up_20230517152807(tx *sql.Tx) error { + _, err := tx.Exec(` +ALTER TABLE hosts ADD COLUMN refetch_critical_queries_until TIMESTAMP NULL; +`) + return errors.Wrap(err, "add refetch_critical_queries_until") +} + +func Down_20230517152807(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/migrations/tables/20230517152807_AddRefetchCriticalQueriesUntilToHosts_test.go b/server/datastore/mysql/migrations/tables/20230517152807_AddRefetchCriticalQueriesUntilToHosts_test.go new file mode 100644 index 0000000000..29562050c8 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20230517152807_AddRefetchCriticalQueriesUntilToHosts_test.go @@ -0,0 +1,79 @@ +package tables + +import ( + "testing" + "time" + + "github.com/fleetdm/fleet/v4/server" + "github.com/stretchr/testify/require" +) + +func TestUp_20230517152807(t *testing.T) { + db := applyUpToPrev(t) + + someString := func() string { + s, err := server.GenerateRandomText(16) + require.NoError(t, err) + return s + } + + insertStmt := ` + INSERT INTO hosts ( + osquery_host_id, + detail_updated_at, + label_updated_at, + policy_updated_at, + node_key, + hostname, + computer_name, + uuid, + platform, + osquery_version, + os_version, + uptime, + memory, + team_id, + distributed_interval, + logger_tls_period, + config_tls_refresh, + refetch_requested, + hardware_serial + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + + newHostArgs := func() []any { + return []any{ + someString(), + time.Now(), + time.Now(), + time.Now(), + someString(), + someString(), + someString(), + someString(), + someString(), + someString(), + someString(), + 1337, + 1337, + nil, + 1337, + 1337, + 1337, + true, + someString(), + } + } + + args := newHostArgs() + execNoErr(t, db, insertStmt, args...) + + // Apply current migration. + applyNext(t, db) + + // existing host has a null refetch_critical_queries_until + var until *time.Time + err := db.Get(&until, "SELECT refetch_critical_queries_until FROM hosts WHERE osquery_host_id = ?", args[0]) + require.NoError(t, err) + require.Nil(t, until) +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index ff6aab629b..609ca34dec 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -412,6 +412,7 @@ CREATE TABLE `hosts` ( `policy_updated_at` timestamp NOT NULL DEFAULT '2000-01-01 00:00:00', `public_ip` varchar(45) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', `orbit_node_key` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `refetch_critical_queries_until` timestamp NULL DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `idx_osquery_host_id` (`osquery_host_id`), UNIQUE KEY `idx_host_unique_nodekey` (`node_key`), @@ -632,9 +633,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=188 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) ENGINE=InnoDB AUTO_INCREMENT=189 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `mobile_device_management_solutions` ( diff --git a/server/fleet/hosts.go b/server/fleet/hosts.go index 8800aa4f10..a5d486eeb0 100644 --- a/server/fleet/hosts.go +++ b/server/fleet/hosts.go @@ -281,6 +281,22 @@ type Host struct { // other host fields, it is not filled in by all host-returning datastore // methods. MDMInfo *HostMDM `json:"-" csv:"-"` + + // RefetchCriticalQueriesUntil can be set to a timestamp up to which the + // "critical" queries will be constantly reported to the host that checks in + // to be re-executed until a condition is met (or the timestamp expires). The + // notion of "critical query" is voluntarily loosely defined so that future + // requirements may use this mechanism. The difference with RefetchRequested + // is that the latter is a one-time request, while this one is a persistent + // until the timestamp expires. The initial use-case is to check for a host + // to be unenrolled from its old MDM solution, in the "migrate to Fleet MDM" + // workflow. + // + // In the future, if we want to use it for more than one use-case, we could + // add a "reason" field with well-known labels so we know what condition(s) + // are expected to clear the timestamp. For now there's a single use-case + // so we don't need this. + RefetchCriticalQueriesUntil *time.Time `json:"-" db:"refetch_critical_queries_until" csv:"-"` } type MDMHostData struct { diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index ceb692d6ee..f2db295a81 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -3172,7 +3172,9 @@ func (s *integrationMDMTestSuite) TestMigrateMDMDeviceWebhook() { h := createHostAndDeviceToken(t, s.ds, "good-token") + var webhookCalled bool webhookSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + webhookCalled = true w.WriteHeader(http.StatusOK) switch r.URL.Path { case "/test_mdm_migration": @@ -3200,7 +3202,7 @@ func (s *integrationMDMTestSuite) TestMigrateMDMDeviceWebhook() { "enable": true, "mode": "voluntary", "webhook_url": "%s/test_mdm_migration" - } + } } }`, webhookSrv.URL)), http.StatusOK, &acResp) require.True(t, acResp.MDM.MacOSMigration.Enable) @@ -3213,25 +3215,63 @@ func (s *integrationMDMTestSuite) TestMigrateMDMDeviceWebhook() { // host is a server so migration is not allowed require.NoError(t, s.ds.SetOrUpdateMDMData(context.Background(), h.ID, isServer, enrolled, mdmURL, installedFromDEP, mdmName)) s.Do("POST", fmt.Sprintf("/api/v1/fleet/device/%s/migrate_mdm", "good-token"), nil, http.StatusBadRequest) + require.False(t, webhookCalled) // host is not DEP so migration is not allowed require.NoError(t, s.ds.SetOrUpdateMDMData(context.Background(), h.ID, !isServer, enrolled, mdmURL, !installedFromDEP, mdmName)) s.Do("POST", fmt.Sprintf("/api/v1/fleet/device/%s/migrate_mdm", "good-token"), nil, http.StatusBadRequest) + require.False(t, webhookCalled) // host is not enrolled to MDM so migration is not allowed require.NoError(t, s.ds.SetOrUpdateMDMData(context.Background(), h.ID, !isServer, !enrolled, mdmURL, installedFromDEP, mdmName)) s.Do("POST", fmt.Sprintf("/api/v1/fleet/device/%s/migrate_mdm", "good-token"), nil, http.StatusBadRequest) + require.False(t, webhookCalled) // host is already enrolled to Fleet MDM so migration is not allowed require.NoError(t, s.ds.SetOrUpdateMDMData(context.Background(), h.ID, !isServer, enrolled, mdmURL, installedFromDEP, fleet.WellKnownMDMFleet)) s.Do("POST", fmt.Sprintf("/api/v1/fleet/device/%s/migrate_mdm", "good-token"), nil, http.StatusBadRequest) + require.False(t, webhookCalled) + + // up to this point, the refetch critical queries timestamp has not been set + // on the host. + h, err := s.ds.Host(context.Background(), h.ID) + require.NoError(t, err) + require.Nil(t, h.RefetchCriticalQueriesUntil) // host is enrolled to a third-party MDM so migration is allowed require.NoError(t, s.ds.SetOrUpdateMDMData(context.Background(), h.ID, !isServer, enrolled, mdmURL, installedFromDEP, mdmName)) s.Do("POST", fmt.Sprintf("/api/v1/fleet/device/%s/migrate_mdm", "good-token"), nil, http.StatusNoContent) + require.True(t, webhookCalled) + webhookCalled = false + + // the refetch critical queries timestamp has been set in the future + h, err = s.ds.Host(context.Background(), h.ID) + require.NoError(t, err) + require.NotNil(t, h.RefetchCriticalQueriesUntil) + require.True(t, h.RefetchCriticalQueriesUntil.After(time.Now())) + + // calling again works but does not trigger the webhook, as it was called recently + s.Do("POST", fmt.Sprintf("/api/v1/fleet/device/%s/migrate_mdm", "good-token"), nil, http.StatusNoContent) + require.False(t, webhookCalled) + + // setting the refetch critical queries timestamp in the past triggers the webhook again + h.RefetchCriticalQueriesUntil = ptr.Time(time.Now().Add(-1 * time.Minute)) + err = s.ds.UpdateHost(context.Background(), h) + require.NoError(t, err) + + s.Do("POST", fmt.Sprintf("/api/v1/fleet/device/%s/migrate_mdm", "good-token"), nil, http.StatusNoContent) + require.True(t, webhookCalled) + webhookCalled = false + + // the refetch critical queries timestamp has been updated to the future + h, err = s.ds.Host(context.Background(), h.ID) + require.NoError(t, err) + require.NotNil(t, h.RefetchCriticalQueriesUntil) + require.True(t, h.RefetchCriticalQueriesUntil.After(time.Now())) // bad token s.Do("POST", fmt.Sprintf("/api/v1/fleet/device/%s/migrate_mdm", "bad-token"), nil, http.StatusUnauthorized) + require.False(t, webhookCalled) // disable macos migration s.DoJSON("PATCH", "/api/latest/fleet/config", json.RawMessage(`{ @@ -3247,6 +3287,7 @@ func (s *integrationMDMTestSuite) TestMigrateMDMDeviceWebhook() { // expect error if macos migration is not configured s.Do("POST", fmt.Sprintf("/api/v1/fleet/device/%s/migrate_mdm", "good-token"), nil, http.StatusBadRequest) + require.False(t, webhookCalled) } func (s *integrationMDMTestSuite) TestMDMMacOSSetup() { diff --git a/server/service/osquery.go b/server/service/osquery.go index 6f43a50e2c..ff066b03da 100644 --- a/server/service/osquery.go +++ b/server/service/osquery.go @@ -608,11 +608,24 @@ func (svc *Service) GetDistributedQueries(ctx context.Context) (queries map[stri const alwaysTrueQuery = "SELECT 1" +// list of detail queries that are returned when only the critical queries +// should be returned (due to RefetchCriticalQueriesUntil timestamp being set). +var criticalDetailQueries = map[string]bool{ + "mdm": true, +} + // detailQueriesForHost returns the map of detail+additional queries that should be executed by // osqueryd to fill in the host details. func (svc *Service) detailQueriesForHost(ctx context.Context, host *fleet.Host) (queries map[string]string, discovery map[string]string, err error) { + var criticalQueriesOnly bool if !svc.shouldUpdate(host.DetailUpdatedAt, svc.config.Osquery.DetailUpdateInterval, host.ID) && !host.RefetchRequested { - return nil, nil, nil + // would not return anything, check if critical queries should be returned + if host.RefetchCriticalQueriesUntil != nil && host.RefetchCriticalQueriesUntil.After(svc.clock.Now()) { + // return only those critical queries + criticalQueriesOnly = true + } else { + return nil, nil, nil + } } appConfig, err := svc.ds.AppConfig(ctx) @@ -630,6 +643,10 @@ func (svc *Service) detailQueriesForHost(ctx context.Context, host *fleet.Host) detailQueries := osquery_utils.GetDetailQueries(ctx, svc.config, appConfig, features) for name, query := range detailQueries { + if criticalQueriesOnly && !criticalDetailQueries[name] { + continue + } + if query.RunsForPlatform(host.Platform) { queryName := hostDetailQueryPrefix + name queries[queryName] = query.Query @@ -641,7 +658,7 @@ func (svc *Service) detailQueriesForHost(ctx context.Context, host *fleet.Host) } } - if features.AdditionalQueries == nil { + if features.AdditionalQueries == nil || criticalQueriesOnly { // No additional queries set return queries, discovery, nil } @@ -833,6 +850,7 @@ func (svc *Service) SubmitDistributedQueryResults( additionalUpdated := false labelResults := map[uint]*bool{} policyResults := map[uint]*bool{} + refetchCriticalSet := host.RefetchCriticalQueriesUntil != nil svc.maybeDebugHost(ctx, host, results, statuses, messages) @@ -936,8 +954,9 @@ func (svc *Service) SubmitDistributedQueryResults( if refetchRequested { host.RefetchRequested = false } + refetchCriticalCleared := refetchCriticalSet && host.RefetchCriticalQueriesUntil == nil - if refetchRequested || detailUpdated { + if refetchRequested || detailUpdated || refetchCriticalCleared { appConfig, err := svc.ds.AppConfig(ctx) if err != nil { logging.WithErr(ctx, err) diff --git a/server/service/osquery_test.go b/server/service/osquery_test.go index 6554fc8667..fedee2c999 100644 --- a/server/service/osquery_test.go +++ b/server/service/osquery_test.go @@ -519,6 +519,7 @@ func verifyDiscovery(t *testing.T, queries, discovery map[string]string) { } func TestHostDetailQueries(t *testing.T) { + ctx := context.Background() ds := new(mock.Store) additional := json.RawMessage(`{"foobar": "select foo", "bim": "bam"}`) ds.AppConfigFunc = func(ctx context.Context) (*fleet.AppConfig, error) { @@ -553,23 +554,26 @@ func TestHostDetailQueries(t *testing.T) { jitterH: make(map[time.Duration]*jitterHashTable), } - queries, discovery, err := svc.detailQueriesForHost(context.Background(), &host) + // detail_updated_at is now, so nothing gets returned by default + queries, discovery, err := svc.detailQueriesForHost(ctx, &host) require.NoError(t, err) assert.Empty(t, queries) verifyDiscovery(t, queries, discovery) // With refetch requested detail queries should be returned host.RefetchRequested = true - queries, discovery, err = svc.detailQueriesForHost(context.Background(), &host) + queries, discovery, err = svc.detailQueriesForHost(ctx, &host) require.NoError(t, err) - assert.NotEmpty(t, queries) + // +2: additional queries: bim, foobar + require.Equal(t, len(expectedDetailQueriesForPlatform(host.Platform))+2, len(queries), distQueriesMapKeys(queries)) verifyDiscovery(t, queries, discovery) host.RefetchRequested = false // Advance the time mockClock.AddTime(1*time.Hour + 1*time.Minute) - queries, discovery, err = svc.detailQueriesForHost(context.Background(), &host) + // all queries returned now that detail udpated at is in the past + queries, discovery, err = svc.detailQueriesForHost(ctx, &host) require.NoError(t, err) // +2: additional queries: bim, foobar require.Equal(t, len(expectedDetailQueriesForPlatform(host.Platform))+2, len(queries), distQueriesMapKeys(queries)) @@ -581,6 +585,31 @@ func TestHostDetailQueries(t *testing.T) { } assert.Equal(t, "bam", queries[hostAdditionalQueryPrefix+"bim"]) assert.Equal(t, "select foo", queries[hostAdditionalQueryPrefix+"foobar"]) + + host.DetailUpdatedAt = mockClock.Now() + + // detail_updated_at is now, so nothing gets returned + queries, discovery, err = svc.detailQueriesForHost(ctx, &host) + require.NoError(t, err) + assert.Empty(t, queries) + verifyDiscovery(t, queries, discovery) + + // setting refetch_critical_queries_until in the past still returns nothing + host.RefetchCriticalQueriesUntil = ptr.Time(mockClock.Now().Add(-1 * time.Minute)) + queries, discovery, err = svc.detailQueriesForHost(ctx, &host) + require.NoError(t, err) + assert.Empty(t, queries) + verifyDiscovery(t, queries, discovery) + + // setting refetch_critical_queries_until in the future returns only the critical queries + host.RefetchCriticalQueriesUntil = ptr.Time(mockClock.Now().Add(1 * time.Minute)) + queries, discovery, err = svc.detailQueriesForHost(ctx, &host) + require.NoError(t, err) + require.Equal(t, len(criticalDetailQueries), len(queries), distQueriesMapKeys(queries)) + for name := range criticalDetailQueries { + assert.Contains(t, queries, hostDetailQueryPrefix+name) + } + verifyDiscovery(t, queries, discovery) } func TestQueriesAndHostFeatures(t *testing.T) { diff --git a/server/service/osquery_utils/queries.go b/server/service/osquery_utils/queries.go index 37dbb44f94..2af9617123 100644 --- a/server/service/osquery_utils/queries.go +++ b/server/service/osquery_utils/queries.go @@ -1250,13 +1250,20 @@ func directIngestMDMMac(ctx context.Context, logger log.Logger, host *fleet.Host } } + mdmSolutionName := deduceMDMNameMacOS(rows[0]) + if !enrolled && installedFromDep && mdmSolutionName != fleet.WellKnownMDMFleet && host.RefetchCriticalQueriesUntil != nil { + // the host was unenrolled from a non-Fleet DEP MDM solution, and the + // refetch critical queries timestamp was set, so clear it. + host.RefetchCriticalQueriesUntil = nil + } + return ds.SetOrUpdateMDMData(ctx, host.ID, false, enrolled, rows[0]["server_url"], installedFromDep, - deduceMDMNameMacOS(rows[0]), + mdmSolutionName, ) }