Allow enrolling fleetd using osquery's instance identifier (#15570)

#14879

- [X] Changes file added for user-visible changes in `changes/` or
`orbit/changes/`.
See [Changes
files](https://fleetdm.com/docs/contributing/committing-changes#changes-files)
for more information.
- [x] Added/updated tests
- [x] Manual QA for all new/changed functionality
  - For Orbit and Fleet Desktop changes:
- [x] Manual QA must be performed in the three main OSs, macOS, Windows
and Linux.
- [x] Auto-update manual QA, from released version of component to new
version (see [tools/tuf/test](../tools/tuf/test/README.md)).

changes/14879-fleetd-host-identifier

@@ -0,0 +1 @@
+* Add `--host-identifier` option to fleetd to allow enrolling with a random identifier instead of the default behavior that uses the hardware UUID. This allows supporting running fleetd on VMs that have the same UUID and/or serial number.

cmd/fleetctl/package.go

@@ -220,6 +220,13 @@ func packageCommand() *cli.Command {
 				EnvVars:     []string{"FLEETCTL_ENABLE_SCRIPTS"},
 				Destination: &opt.EnableScripts,
 			},
+			&cli.StringFlag{
+				Name:        "host-identifier",
+				Usage:       "Sets the host identifier that orbit and osquery will use when enrolling to Fleet. Options: 'uuid' and 'instance' (requires Fleet >= v4.42.0)",
+				Value:       "uuid",
+				EnvVars:     []string{"FLEETCTL_HOST_IDENTIFIER"},
+				Destination: &opt.HostIdentifier,
+			},
 		},
 		Action: func(c *cli.Context) error {
 			if opt.FleetURL != "" || opt.EnrollSecret != "" {
@@ -236,6 +243,10 @@ func packageCommand() *cli.Command {
 				return errors.New("--insecure and --update-tls-certificate may not be provided together")
 			}
 
+			if opt.HostIdentifier != "uuid" && opt.HostIdentifier != "instance" {
+				return fmt.Errorf("--host-identifier=%s is not supported, currently supported values are 'uuid' and 'instance'", opt.HostIdentifier)
+			}
+
 			// Perform checks on the provided fleet client certificate and key.
 			if (opt.FleetTLSClientCertificate != "") != (opt.FleetTLSClientKey != "") {
 				return errors.New("must specify both fleet-tls-client-certificate and fleet-tls-client-key")

docs/Using Fleet/fleetd.md

@@ -124,6 +124,7 @@ The following command-line flags allow you to configure an osquery installer fur
 | --update-roots             | Root key JSON metadata for update server (from fleetctl updates roots)                                                                  |
 | --use-system-configuration | Try to read --fleet-url and --enroll-secret using configuration in the host (currently only macOS profiles are supported)               |
 | --enable-scripts           | Enable script execution (default: `false`)                                                                                              |
+| --host-identifier          | Sets the host identifier that orbit and osquery will use when enrolling to Fleet. Options: `uuid` and `instance` (requires Fleet >= v4.42.0) (default: `uuid`)                                                 |
 | --debug                    | Enable debug logging (default: `false`)                                                                                                 |
 | --verbose                  | Log detailed information when building the package (default: false)                                                                     |
 | --help, -h                 | show help (default: `false`)                                                                                                            |

orbit/changes/14879-fleetd-host-identifier

@@ -0,0 +1 @@
+* Add `--host-identifier` option to fleetd to allow enrolling with a random identifier instead of the default behavior that uses the hardware UUID. This allows supporting running fleetd on VMs that have the same UUID and/or serial number.

orbit/cmd/orbit/orbit.go

@@ -177,6 +177,12 @@ func main() {
 			Usage:   "Enable script execution",
 			EnvVars: []string{"ORBIT_ENABLE_SCRIPTS"},
 		},
+		&cli.StringFlag{
+			Name:    "host-identifier",
+			Usage:   "Sets the host identifier that orbit and osquery will use when enrolling to Fleet. Options: 'uuid' and 'instance' (requires Fleet >= v4.42.0)",
+			EnvVars: []string{"ORBIT_HOST_IDENTIFIER"},
+			Value:   "uuid",
+		},
 	}
 	app.Before = func(c *cli.Context) error {
 		// handle old installations, which had default root dir set to /var/lib/orbit
@@ -258,6 +264,10 @@ func main() {
 			}
 		}
 
+		if hostIdentifier := c.String("host-identifier"); hostIdentifier != "uuid" && hostIdentifier != "instance" {
+			return fmt.Errorf("--host-identifier=%s is not supported, currently supported values are 'uuid' and 'instance'", hostIdentifier)
+		}
+
 		if err := secure.MkdirAll(c.String("root-dir"), constant.DefaultDirMode); err != nil {
 			return fmt.Errorf("initialize root dir: %w", err)
 		}
@@ -484,13 +494,35 @@ func main() {
 			return fmt.Errorf("cleanup old files: %w", err)
 		}
 
-		orbitHostInfo, err := getHostInfo(osquerydPath, filepath.Join(c.String("root-dir"), "osquery.db"))
+		osqueryHostInfo, err := getHostInfo(osquerydPath, filepath.Join(c.String("root-dir"), "osquery.db"))
 		if err != nil {
 			return fmt.Errorf("get UUID: %w", err)
 		}
-		log.Debug().Str("info", fmt.Sprint(orbitHostInfo)).Msg("retrieved host info")
+		log.Debug().Str("info", fmt.Sprint(osqueryHostInfo)).Msg("retrieved host info from osquery")
+		orbitHostInfo := fleet.OrbitHostInfo{
+			HardwareSerial: osqueryHostInfo.HardwareSerial,
+			HardwareUUID:   osqueryHostInfo.HardwareUUID,
+			Hostname:       osqueryHostInfo.Hostname,
+			Platform:       osqueryHostInfo.Platform,
+		}
 
-		var options []osquery.Option
+		// Only send osquery's `instance_id` if the user is running orbit with `--host-identifier=instance`.
+		// When not set, orbit and osquery will be matched using the hardware UUID (orbitHostInfo.HardwareUUID).
+		if c.String("host-identifier") == "instance" {
+			orbitHostInfo.OsqueryIdentifier = osqueryHostInfo.InstanceID
+		}
+
+		// The hardware serial was not sent when Windows MDM was implemented,
+		// thus we clear its value here to not break any existing enroll functionality
+		// on the server.
+		if runtime.GOOS == "windows" {
+			orbitHostInfo.HardwareSerial = ""
+		}
+
+		var (
+			options              []osquery.Option
+			optionsAfterFlagfile []osquery.Option
+		)
 		options = append(options, osquery.WithDataPath(c.String("root-dir")))
 		options = append(options, osquery.WithLogPath(filepath.Join(c.String("root-dir"), "osquery_log")))
 
@@ -709,7 +741,8 @@ func main() {
 			case err == nil:
 				if stat.Size() > 0 {
 					log.Debug().Msg("adding --extensions_autoload flag for file " + extensionAutoLoadFile)
-					options = append(options, osquery.WithFlags([]string{"--extensions_autoload", extensionAutoLoadFile}))
+					// We set this option after the --flagfile to prevent users from changing it on their flagfiles.
+					optionsAfterFlagfile = append(optionsAfterFlagfile, osquery.WithFlags([]string{"--extensions_autoload", extensionAutoLoadFile}))
 				} else {
 					// OK, expected as well when extensions are unloaded, just debug log
 					log.Debug().Msg("found empty extensions.load file at " + extensionAutoLoadFile)
@@ -857,6 +890,14 @@ func main() {
 			options = append(options, osquery.WithFlags([]string{"--flagfile", flagfilePath}))
 		}
 
+		// These options must go after '--flagfile' to not allow users to change their values
+		// on their flagfiles.
+		hostIdentifier := c.String("host-identifier")
+		options = append(options, osquery.WithFlags([]string{"--host-identifier", hostIdentifier}))
+		for _, option := range optionsAfterFlagfile {
+			options = append(options, option)
+		}
+
 		// Handle additional args after '--' in the command line. These are added last and should
 		// override all other flags and flagfile entries.
 		options = append(options, osquery.WithFlags(c.Args().Slice()))
@@ -1122,8 +1163,8 @@ func (d *desktopRunner) interrupt(err error) {
 	}
 }
 
-// hostInfo is used to parse osquery JSON output from `system_info` and `os_version` tables.
-type hostInfo struct {
+// osqueryHostInfo is used to parse osquery JSON output from system tables.
+type osqueryHostInfo struct {
 	// HardwareUUID is the unique identifier for this device (extracted from `system_info` osquery table).
 	HardwareUUID string `json:"uuid"`
 	// HardwareSerial is the unique serial number for this device (extracted from `system_info` osquery table).
@@ -1132,47 +1173,14 @@ type hostInfo struct {
 	Hostname string `json:"hostname"`
 	// Platform is the device's platform as defined by osquery (extracted from `os_version` osquery table).
 	Platform string `json:"platform"`
+	// InstanceID is the osquery's randomly generated instance ID
+	// (extracted from `osquery_info` osquery table).
+	InstanceID string `json:"instance_id"`
 }
 
-// getHostInfo retrieves system information about the host.
-//
-// On macOS and Linux it shells out to osqueryd to retrieve the information.
-//
-// On Windows:
-//
-//   - HardwareUUID is retrieved by shelling out to wmic, if that fails
-//     then the windows API are used.
-//   - HardwareSerial is currently not retrieved for Windows devices.
-//   - Hostname is retrieved using stdlib method.
-//   - Platform is always "windows" for windows hosts.
-//
-// NOTE: Windows uses a different approach to retrieve the device information
-// as there were issues at the time with shelling out to osquery - from what the
-// team remembers it would sometimes fail due to the osquery process not being ready yet.
-// A recent CI run without the Windows special-case did succeed, but since we don't
-// need the serial number for Windows at the moment, we opted to keep the
-// code as it is.
-func getHostInfo(osqueryPath string, osqueryDBPath string) (fleet.OrbitHostInfo, error) {
-	if runtime.GOOS == "windows" {
-		uuidData, uuidSource, err := platform.GetSMBiosUUID()
-		if err != nil {
-			return fleet.OrbitHostInfo{}, err
-		}
-		log.Debug().Str("source", string(uuidSource)).Msg("UUID")
-		// Hostname might differ from the one provided by osquery but we are sending it
-		// for troubleshooting purposes and to avoid empty host entries in the UI.
-		hostname, err := os.Hostname()
-		if err != nil {
-			return fleet.OrbitHostInfo{}, err
-		}
-		return fleet.OrbitHostInfo{
-			HardwareUUID:   uuidData,
-			HardwareSerial: "", // currently not needed for Windows.
-			Hostname:       hostname,
-			Platform:       "windows",
-		}, nil
-	}
-	const systemQuery = "SELECT si.uuid, si.hardware_serial, si.hostname, os.platform FROM system_info si, os_version os"
+// getHostInfo retrieves system information about the host by shelling out to `osqueryd -S` and performing a `SELECT` query.
+func getHostInfo(osqueryPath string, osqueryDBPath string) (*osqueryHostInfo, error) {
+	const systemQuery = "SELECT si.uuid, si.hardware_serial, si.hostname, os.platform, oi.instance_id FROM system_info si, os_version os, osquery_info oi"
 	args := []string{
 		"-S",
 		"--database_path", osqueryDBPath,
@@ -1181,22 +1189,17 @@ func getHostInfo(osqueryPath string, osqueryDBPath string) (fleet.OrbitHostInfo,
 	log.Debug().Str("query", systemQuery).Msg("running single query")
 	out, err := exec.Command(osqueryPath, args...).Output()
 	if err != nil {
-		return fleet.OrbitHostInfo{}, err
+		return nil, err
 	}
-	var info []hostInfo
+	var info []osqueryHostInfo
 	err = json.Unmarshal(out, &info)
 	if err != nil {
-		return fleet.OrbitHostInfo{}, err
+		return nil, err
 	}
 	if len(info) != 1 {
-		return fleet.OrbitHostInfo{}, fmt.Errorf("invalid number of rows from system info query: %d", len(info))
+		return nil, fmt.Errorf("invalid number of rows from system info query: %d", len(info))
 	}
-	return fleet.OrbitHostInfo{
-		HardwareSerial: info[0].HardwareSerial,
-		HardwareUUID:   info[0].HardwareUUID,
-		Hostname:       info[0].Hostname,
-		Platform:       info[0].Platform,
-	}, nil
+	return &info[0], nil
 }
 
 var versionCommand = &cli.Command{

orbit/pkg/osquery/flags.go

@@ -9,8 +9,6 @@ import (
 func FleetFlags(fleetURL *url.URL) []string {
 	hostname, prefix := fleetURL.Host, fleetURL.Path
 	return []string{
-		// Use uuid as the default identifier -- users can override this in their flagfile
-		"--host_identifier=uuid",
 		"--tls_hostname=" + hostname,
 		"--enroll_tls_endpoint=" + path.Join(prefix, "/api/v1/osquery/enroll"),
 		"--config_plugin=tls",

orbit/pkg/packaging/linux_shared.go

@@ -293,6 +293,7 @@ ORBIT_FLEET_DESKTOP_ALTERNATIVE_BROWSER_HOST={{ .FleetDesktopAlternativeBrowserH
 {{ if .EnrollSecret }}ORBIT_ENROLL_SECRET={{.EnrollSecret}}{{ end }}
 {{ if .Debug }}ORBIT_DEBUG=true{{ end }}
 {{ if .EnableScripts }}ORBIT_ENABLE_SCRIPTS=true{{ end }}
+{{ if and (ne .HostIdentifier "") (ne .HostIdentifier "uuid") }}ORBIT_HOST_IDENTIFIER={{.HostIdentifier}}{{ end }}
 `))
 
 func writeEnvFile(opt Options, rootPath string) error {

orbit/pkg/packaging/macos_templates.go

@@ -88,8 +88,6 @@ launchctl kickstart "system/${DAEMON_LABEL}"
 {{- end }}
 `))
 
-// TODO set Nice?
-//
 // Note it's important not to start the orbit binary in
 // `/usr/local/bin/orbit` because this is a path that users usually have write
 // access to, and running that binary with launchd can become a privilege
@@ -155,6 +153,10 @@ var macosLaunchdTemplate = template.Must(template.New("").Option("missingkey=err
 		{{- end }}
 		<key>ORBIT_UPDATE_INTERVAL</key>
 		<string>{{ .OrbitUpdateInterval }}</string>
+		{{- if and (ne .HostIdentifier "") (ne .HostIdentifier "uuid") }}
+		<key>ORBIT_HOST_IDENTIFIER</key>
+		<string>{{ .HostIdentifier }}</string>
+		{{- end }}
 	</dict>
 	<key>KeepAlive</key>
 	<true/>

orbit/pkg/packaging/packaging.go

@@ -115,6 +115,8 @@ type Options struct {
 	// LocalWixDir uses a Windows machine's local WiX installation instead of a containerized
 	// emulation to build an MSI fleetd installer
 	LocalWixDir string
+	// HostIdentifier is the host identifier to use in osquery.
+	HostIdentifier string
 }
 
 func initializeTempDir() (string, error) {

orbit/pkg/packaging/windows_templates.go

@@ -99,7 +99,7 @@ var windowsWixTemplate = template.Must(template.New("").Option("missingkey=error
                   Start="auto"
                   Type="ownProcess"
                   Description="This service runs Fleet's osquery runtime and autoupdater (Orbit)."
-                  Arguments='--root-dir "[ORBITROOT]." --log-file "[System64Folder]config\systemprofile\AppData\Local\FleetDM\Orbit\Logs\orbit-osquery.log" --fleet-url "[FLEET_URL]"{{ if .FleetCertificate }} --fleet-certificate "[ORBITROOT]fleet.pem"{{ end }}{{ if .EnrollSecret }} --enroll-secret-path "[ORBITROOT]secret.txt"{{ end }}{{if .Insecure }} --insecure{{ end }}{{ if .Debug }} --debug{{ end }}{{ if .UpdateURL }} --update-url "{{ .UpdateURL }}"{{ end }}{{ if .UpdateTLSServerCertificate }} --update-tls-certificate "[ORBITROOT]update.pem"{{ end }}{{ if .DisableUpdates }} --disable-updates{{ end }}{{ if .Desktop }} --fleet-desktop --desktop-channel {{ .DesktopChannel }}{{ if .FleetDesktopAlternativeBrowserHost }} --fleet-desktop-alternative-browser-host {{ .FleetDesktopAlternativeBrowserHost }}{{ end }}{{ end }} --orbit-channel "{{ .OrbitChannel }}" --osqueryd-channel "{{ .OsquerydChannel }}" {{ if .EnableScripts }} --enable-scripts{{ end }}'
+                  Arguments='--root-dir "[ORBITROOT]." --log-file "[System64Folder]config\systemprofile\AppData\Local\FleetDM\Orbit\Logs\orbit-osquery.log" --fleet-url "[FLEET_URL]"{{ if .FleetCertificate }} --fleet-certificate "[ORBITROOT]fleet.pem"{{ end }}{{ if .EnrollSecret }} --enroll-secret-path "[ORBITROOT]secret.txt"{{ end }}{{if .Insecure }} --insecure{{ end }}{{ if .Debug }} --debug{{ end }}{{ if .UpdateURL }} --update-url "{{ .UpdateURL }}"{{ end }}{{ if .UpdateTLSServerCertificate }} --update-tls-certificate "[ORBITROOT]update.pem"{{ end }}{{ if .DisableUpdates }} --disable-updates{{ end }}{{ if .Desktop }} --fleet-desktop --desktop-channel {{ .DesktopChannel }}{{ if .FleetDesktopAlternativeBrowserHost }} --fleet-desktop-alternative-browser-host {{ .FleetDesktopAlternativeBrowserHost }}{{ end }}{{ end }} --orbit-channel "{{ .OrbitChannel }}" --osqueryd-channel "{{ .OsquerydChannel }}" {{ if .EnableScripts }} --enable-scripts{{ end }}{{ if and (ne .HostIdentifier "") (ne .HostIdentifier "uuid") }}--host-identifier={{ .HostIdentifier }}{{ end }}'
                 >
                   <util:ServiceConfig
                     FirstFailureActionType="restart"

server/datastore/mysql/apple_mdm.go

@@ -577,7 +577,7 @@ func ingestMDMAppleDeviceFromCheckinDB(
 
 	// MDM is necessarily enabled if this gets called, always pass true for that
 	// parameter.
-	matchID, _, err := matchHostDuringEnrollment(ctx, tx, true, "", mdmHost.UDID, mdmHost.SerialNumber)
+	matchID, _, err := matchHostDuringEnrollment(ctx, tx, mdmEnroll, true, "", mdmHost.UDID, mdmHost.SerialNumber)
 	switch {
 	case errors.Is(err, sql.ErrNoRows):
 		return insertMDMAppleHostDB(ctx, tx, mdmHost, logger, appCfg)

server/datastore/mysql/apple_mdm_test.go

@@ -4746,7 +4746,12 @@ func TestRestorePendingDEPHost(t *testing.T) {
 			require.WithinDuration(t, time.Now(), depAssignment.AddedAt, 5*time.Second)
 
 			// simulate initial osquery enrollment via Orbit
-			h, err := ds.EnrollOrbit(ctx, true, fleet.OrbitHostInfo{HardwareSerial: depSerial, Platform: "darwin", HardwareUUID: depUUID, Hostname: "dep-host"}, depOrbitNodeKey, nil)
+			h, err := ds.EnrollOrbit(ctx, true, fleet.OrbitHostInfo{
+				HardwareSerial: depSerial,
+				Platform:       "darwin",
+				HardwareUUID:   depUUID,
+				Hostname:       "dep-host",
+			}, depOrbitNodeKey, nil)
 			require.NoError(t, err)
 			require.NotNil(t, h)
 			require.Equal(t, depHostID, h.ID)

server/datastore/mysql/hosts.go

@@ -1609,12 +1609,19 @@ func (ds *Datastore) GenerateHostStatusStatistics(ctx context.Context, filter fl
 	return &summary, nil
 }
 
+type enroll uint
+
+const (
+	osqueryEnroll enroll = iota
+	orbitEnroll
+	mdmEnroll
+)
+
 // Attempts to find the matching host ID by osqueryID, host UUID or serial
 // number. Any of those fields can be left empty if not available, and it will
 // use the best match in this order:
 // * if it matched on osquery_host_id (with osqueryID or uuid), use that host
-// * otherwise if it matched on uuid, use that host
-// * otherwise use the match on serial
+// * otherwise if it matched on serial, use that host
 //
 // Note that in general, all options should result in a single match anyway.
 // It's just that our DB schema doesn't enforce this (only osquery_host_id has
@@ -1624,7 +1631,7 @@ func (ds *Datastore) GenerateHostStatusStatistics(ctx context.Context, filter fl
 // able to match by serial in this scenario, since this is the only information
 // we get when enrolling hosts via Apple DEP) AND if the matched host is on the
 // macOS platform (darwin).
-func matchHostDuringEnrollment(ctx context.Context, q sqlx.QueryerContext, isMDMEnabled bool, osqueryID, uuid, serial string) (uint, time.Time, error) {
+func matchHostDuringEnrollment(ctx context.Context, q sqlx.QueryerContext, enrollType enroll, isMDMEnabled bool, osqueryID, uuid, serial string) (uint, time.Time, error) {
 	type hostMatch struct {
 		ID             uint
 		LastEnrolledAt time.Time `db:"last_enrolled_at"`
@@ -1639,32 +1646,22 @@ func matchHostDuringEnrollment(ctx context.Context, q sqlx.QueryerContext, isMDM
 
 	if osqueryID != "" || uuid != "" {
 		_, _ = query.WriteString(`(SELECT id, last_enrolled_at, 1 priority FROM hosts WHERE osquery_host_id = ?)`)
+		osqueryHostID := osqueryID
 		if osqueryID == "" {
 			// special-case, if there's no osquery identifier, use the uuid
-			osqueryID = uuid
+			osqueryHostID = uuid
 		}
-		args = append(args, osqueryID)
+		args = append(args, osqueryHostID)
 	}
 
-	// TODO(mna): for now do not match by UUID on the `uuid` field as it is not indexed.
-	// See https://github.com/fleetdm/fleet/issues/9372 and
-	// https://github.com/fleetdm/fleet/issues/9033#issuecomment-1411150758
-	// (the latter shows that it might not be top priority to index this field, if we're
-	// going to recommend using the host uuid as osquery identifier, as osquery_host_id
-	// _is_ indexed and unique).
-	// if uuid != "" {
-	//	if query.Len() > 0 {
-	//		_, _ = query.WriteString(" UNION ")
-	//	}
-	//	_, _ = query.WriteString(`(SELECT id, last_enrolled_at, 2 priority FROM hosts WHERE uuid = ? ORDER BY id LIMIT 1)`)
-	//	args = append(args, uuid)
-	// }
+	// We want to prevent orbit enrolling with an osquery identifier to be matched with the serial number.
+	orbitEnrollingWithOsqueryIdentifier := enrollType == orbitEnroll && osqueryID != ""
 
-	if serial != "" && isMDMEnabled {
+	if serial != "" && isMDMEnabled && !orbitEnrollingWithOsqueryIdentifier {
 		if query.Len() > 0 {
 			_, _ = query.WriteString(" UNION ")
 		}
-		_, _ = query.WriteString(`(SELECT id, last_enrolled_at, 3 priority FROM hosts WHERE hardware_serial = ? AND platform = ? ORDER BY id LIMIT 1)`)
+		_, _ = query.WriteString(`(SELECT id, last_enrolled_at, 2 priority FROM hosts WHERE hardware_serial = ? AND platform = ? ORDER BY id LIMIT 1)`)
 		args = append(args, serial, "darwin")
 	}
 
@@ -1692,7 +1689,15 @@ func (ds *Datastore) EnrollOrbit(ctx context.Context, isMDMEnabled bool, hostInf
 
 	var host fleet.Host
 	err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
-		hostID, _, err := matchHostDuringEnrollment(ctx, tx, isMDMEnabled, "", hostInfo.HardwareUUID, hostInfo.HardwareSerial)
+		hostID, _, err := matchHostDuringEnrollment(ctx, tx, orbitEnroll, isMDMEnabled, hostInfo.OsqueryIdentifier, hostInfo.HardwareUUID, hostInfo.HardwareSerial)
+
+		// If the osquery identifier that osqueryd will use was not sent by Orbit, then use the hardware UUID as identifier
+		// (using the hardware UUID is Orbit's default behavior).
+		osqueryIdentifier := hostInfo.OsqueryIdentifier
+		if osqueryIdentifier == "" {
+			osqueryIdentifier = hostInfo.HardwareUUID
+		}
+
 		switch {
 		case err == nil:
 			sqlUpdate := `
@@ -1708,7 +1713,7 @@ func (ds *Datastore) EnrollOrbit(ctx context.Context, isMDMEnabled bool, hostInf
 			_, err := tx.ExecContext(ctx, sqlUpdate,
 				orbitNodeKey,
 				hostInfo.HardwareUUID,
-				hostInfo.HardwareUUID,
+				osqueryIdentifier,
 				hostInfo.HardwareSerial,
 				teamID,
 				hostID,
@@ -1747,7 +1752,7 @@ func (ds *Datastore) EnrollOrbit(ctx context.Context, isMDMEnabled bool, hostInf
 				zeroTime,
 				zeroTime,
 				zeroTime,
-				hostInfo.HardwareUUID,
+				osqueryIdentifier,
 				hostInfo.HardwareUUID,
 				orbitNodeKey,
 				teamID,
@@ -1791,7 +1796,7 @@ func (ds *Datastore) EnrollHost(ctx context.Context, isMDMEnabled bool, osqueryH
 	err := ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
 		zeroTime := time.Unix(0, 0).Add(24 * time.Hour)
 
-		matchedID, lastEnrolledAt, err := matchHostDuringEnrollment(ctx, tx, isMDMEnabled, osqueryHostID, hardwareUUID, hardwareSerial)
+		matchedID, lastEnrolledAt, err := matchHostDuringEnrollment(ctx, tx, osqueryEnroll, isMDMEnabled, osqueryHostID, hardwareUUID, hardwareSerial)
 		switch {
 		case err != nil && !errors.Is(err, sql.ErrNoRows):
 			return ctxerr.Wrap(ctx, err, "check existing")

server/datastore/mysql/hosts_test.go

@@ -7330,7 +7330,7 @@ func testHostsEnrollOrbit(t *testing.T, ds *Datastore) {
 	}, uuid.New().String(), nil)
 	require.NoError(t, err)
 	require.Equal(t, hBoth.ID, h.ID)
-	require.Empty(t, h.HardwareSerial) // this is just to prove that it was loaded based on osquery_node_id, the serial was not set in the lookup
+	require.Empty(t, h.HardwareSerial) // this is just to prove that it was loaded based on osquery_host_id, the serial was not set in the lookup
 
 	// enroll with osquery id from hBoth and serial from hSerialNoOsquery (should
 	// use the osquery match)
@@ -7377,6 +7377,150 @@ func testHostsEnrollOrbit(t *testing.T, ds *Datastore) {
 	}, uuid.New().String(), nil)
 	require.NoError(t, err)
 	require.Equal(t, hOsqueryNoSerial.ID, h.ID)
+
+	// Scenario A:
+	//	- Fleet with MDM disabled.
+	// 	- two linux|darwin|windows hosts with the same hardware identifiers (e.g. two cloned VMs).
+	//	- fleetd running with host identifier set to instance.
+	//	- orbit enrolls first, then osquery
+	// Expected output: The two fleetd instances should be enrolled as two hosts.
+	scenarioA := func(platform string) {
+		dupUUID := uuid.New().String()
+		dupHWSerial := uuid.New().String()
+		randomIdentifierH1 := uuid.New().String()
+
+		h1Orbit, err := ds.EnrollOrbit(ctx, false, fleet.OrbitHostInfo{
+			HardwareUUID:      dupUUID,
+			HardwareSerial:    dupHWSerial,
+			OsqueryIdentifier: randomIdentifierH1,
+			Platform:          platform,
+		}, uuid.New().String(), nil)
+		require.NoError(t, err)
+		h1Osquery, err := ds.EnrollHost(ctx, false, randomIdentifierH1, dupUUID, dupHWSerial, uuid.New().String(), nil, 0)
+		require.NoError(t, err)
+		require.Equal(t, h1Orbit.ID, h1Osquery.ID)
+		randomIdentifierH2 := uuid.New().String()
+		h2Orbit, err := ds.EnrollOrbit(ctx, false, fleet.OrbitHostInfo{
+			HardwareUUID:      dupUUID,
+			HardwareSerial:    dupHWSerial,
+			OsqueryIdentifier: randomIdentifierH2,
+			Platform:          platform,
+		}, uuid.New().String(), nil)
+		require.NoError(t, err)
+		h2Osquery, err := ds.EnrollHost(ctx, false, randomIdentifierH2, dupUUID, dupHWSerial, uuid.New().String(), nil, 0)
+		require.NoError(t, err)
+		require.Equal(t, h2Orbit.ID, h2Osquery.ID)
+
+		require.NotEqual(t, h1Orbit.ID, h2Orbit.ID) // the hosts are enrolled as two separate hosts
+	}
+	for _, platform := range []string{"ubuntu", "windows", "darwin"} {
+		platform := platform
+		t.Run("scenarioA_"+platform, func(t *testing.T) {
+			scenarioA(platform)
+		})
+	}
+
+	// Scenario B:
+	//	- Fleet with MDM disabled.
+	// 	- Two linux|darwin|windows hosts with the same hardware identifiers (e.g. two cloned VMs).
+	//	- fleetd running with host identifier set to instance.
+	//	- orbit and osquery of the two hosts enroll in mixed order.
+	// Expected output: The two fleetd instances should be each its own host.
+	scenarioB := func(platform string) {
+		dupUUID := uuid.New().String()
+		dupHWSerial := uuid.New().String()
+		randomIdentifierH1 := uuid.New().String()
+
+		// First osquery of the first host enrolls.
+		h1Osquery, err := ds.EnrollHost(ctx, false, randomIdentifierH1, dupUUID, dupHWSerial, uuid.New().String(), nil, 0)
+		require.NoError(t, err)
+		randomIdentifierH2 := uuid.New().String()
+		// Then orbit of the second host enrolls.
+		h2Orbit, err := ds.EnrollOrbit(ctx, false, fleet.OrbitHostInfo{
+			HardwareUUID:      dupUUID,
+			HardwareSerial:    dupHWSerial,
+			OsqueryIdentifier: randomIdentifierH2,
+			Platform:          platform,
+		}, uuid.New().String(), nil)
+		require.NoError(t, err)
+		// Then orbit of the first host enrolls.
+		h1Orbit, err := ds.EnrollOrbit(ctx, false, fleet.OrbitHostInfo{
+			HardwareUUID:      dupUUID,
+			HardwareSerial:    dupHWSerial,
+			OsqueryIdentifier: randomIdentifierH1,
+			Platform:          platform,
+		}, uuid.New().String(), nil)
+		require.NoError(t, err)
+		require.Equal(t, h1Orbit.ID, h1Osquery.ID)
+		// Lastly osquery of the second host enrolls.
+		h2Osquery, err := ds.EnrollHost(ctx, false, randomIdentifierH2, dupUUID, dupHWSerial, uuid.New().String(), nil, 0)
+		require.NoError(t, err)
+		require.Equal(t, h2Orbit.ID, h2Osquery.ID)
+
+		require.NotEqual(t, h1Orbit.ID, h2Orbit.ID) // the hosts are enrolled as two separate hosts
+	}
+	for _, platform := range []string{"ubuntu", "windows", "darwin"} {
+		platform := platform
+		t.Run("scenarioB_"+platform, func(t *testing.T) {
+			scenarioB(platform)
+		})
+	}
+
+	// Scenario C:
+	//	- Fleet with MDM enabled.
+	// 	- Two linux|darwin|windows hosts with the same hardware identifiers (e.g. two cloned VMs).
+	//	- fleetd running with host identifier set to instance.
+	//	- orbit and osquery of the two hosts enroll in mixed order.
+	//
+	// For Linux and Windows this scenario behaves as expected. The two hosts are enrolled separately.
+	//
+	// For macOS:
+	// Somewhat unexpected output of this scenario is that two hosts are enrolled as one
+	// because MDM makes the effort to match by hardware serial.
+	// Using fleetd's `--host-identifier=instance` with Fleet's MDM enabled is not compatible on macOS.
+	scenarioC := func(platform string) {
+		dupUUID := uuid.New().String()
+		dupHWSerial := uuid.New().String()
+		randomIdentifierH1 := uuid.New().String()
+		randomIdentifierH2 := uuid.New().String()
+
+		h1Orbit, err := ds.EnrollOrbit(ctx, true, fleet.OrbitHostInfo{
+			HardwareUUID:      dupUUID,
+			HardwareSerial:    dupHWSerial,
+			OsqueryIdentifier: randomIdentifierH1,
+			Platform:          platform,
+		}, uuid.New().String(), nil)
+		require.NoError(t, err)
+		h1Osquery, err := ds.EnrollHost(ctx, true, randomIdentifierH1, dupUUID, dupHWSerial, uuid.New().String(), nil, 0)
+		require.NoError(t, err)
+		require.Equal(t, h1Orbit.ID, h1Osquery.ID)
+
+		// Second host enrolls osquery first, then orbit.
+		h2Osquery, err := ds.EnrollHost(ctx, true, randomIdentifierH2, dupUUID, dupHWSerial, uuid.New().String(), nil, 0)
+		require.NoError(t, err)
+		h2Orbit, err := ds.EnrollOrbit(ctx, true, fleet.OrbitHostInfo{
+			HardwareUUID:      dupUUID,
+			HardwareSerial:    dupHWSerial,
+			OsqueryIdentifier: randomIdentifierH2,
+			Platform:          platform,
+		}, uuid.New().String(), nil)
+		require.NoError(t, err)
+		require.Equal(t, h2Orbit.ID, h2Osquery.ID)
+
+		if platform == "darwin" {
+			// This is a expected output of this scenario because MDM makes
+			// the effort to match by hardware serial.
+			require.Equal(t, h1Orbit.ID, h2Orbit.ID)
+		} else {
+			require.NotEqual(t, h1Orbit.ID, h2Orbit.ID)
+		}
+	}
+	for _, platform := range []string{"ubuntu", "windows", "darwin"} {
+		platform := platform
+		t.Run("scenarioC_"+platform, func(t *testing.T) {
+			scenarioC(platform)
+		})
+	}
 }
 
 func testHostsEnrollUpdatesMissingInfo(t *testing.T, ds *Datastore) {

server/fleet/agent_options.go

@@ -50,6 +50,15 @@ func ValidateJSONAgentOptions(ctx context.Context, ds Datastore, rawJSON json.Ra
 		if err := JSONStrictDecode(bytes.NewReader(opts.CommandLineStartUpFlags), &flags); err != nil {
 			return fmt.Errorf("command-line flags: %w", err)
 		}
+
+		// We prevent setting the following flags because they can break fleetd.
+		flagNotSupportedErr := "The %s flag isn't supported. Please remove this flag."
+		if flags.HostIdentifier != "" {
+			return fmt.Errorf(flagNotSupportedErr, "--host_identifier")
+		}
+		if flags.ExtensionsAutoload != "" {
+			return fmt.Errorf(flagNotSupportedErr, "--extensions_autoload")
+		}
 	}
 
 	if len(opts.Config) > 0 {

server/fleet/orbit.go

@@ -53,6 +53,12 @@ type OrbitHostInfo struct {
 	Hostname string
 	// Platform is the device's platform as defined by osquery.
 	Platform string
+	// OsqueryIdentifier holds the identifier that osqueryd will use in its enrollment.
+	// This is mainly used for scenarios where hosts have duplicate hardware UUID (e.g. VMs)
+	// and a different identifier is used for each host (e.g. osquery's "instance" flag).
+	//
+	// If not set, then the HardwareUUID is used/set as the osquery identifier.
+	OsqueryIdentifier string
 }
 
 // ExtensionInfo holds the data of a osquery extension to apply to an Orbit client.

server/service/orbit.go

@@ -13,10 +13,9 @@ import (
 	"github.com/fleetdm/fleet/v4/server/contexts/license"
 	"github.com/fleetdm/fleet/v4/server/contexts/logging"
 	"github.com/fleetdm/fleet/v4/server/fleet"
+	microsoft_mdm "github.com/fleetdm/fleet/v4/server/mdm/microsoft"
 	"github.com/fleetdm/fleet/v4/server/ptr"
 	"github.com/go-kit/kit/log/level"
-
-	microsoft_mdm "github.com/fleetdm/fleet/v4/server/mdm/microsoft"
 )
 
 type setOrbitNodeKeyer interface {
@@ -35,6 +34,9 @@ type EnrollOrbitRequest struct {
 	Hostname string `json:"hostname"`
 	// Platform is the device's platform as defined by osquery.
 	Platform string `json:"platform"`
+	// OsqueryIdentifier holds the identifier used by osquery.
+	// If not set, then the hardware UUID is used to match orbit and osquery.
+	OsqueryIdentifier string `json:"osquery_identifier"`
 }
 
 type EnrollOrbitResponse struct {
@@ -79,10 +81,11 @@ func (r EnrollOrbitResponse) hijackRender(ctx context.Context, w http.ResponseWr
 func enrollOrbitEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) {
 	req := request.(*EnrollOrbitRequest)
 	nodeKey, err := svc.EnrollOrbit(ctx, fleet.OrbitHostInfo{
-		HardwareUUID:   req.HardwareUUID,
-		HardwareSerial: req.HardwareSerial,
-		Hostname:       req.Hostname,
-		Platform:       req.Platform,
+		HardwareUUID:      req.HardwareUUID,
+		HardwareSerial:    req.HardwareSerial,
+		Hostname:          req.Hostname,
+		Platform:          req.Platform,
+		OsqueryIdentifier: req.OsqueryIdentifier,
 	}, req.EnrollSecret)
 	if err != nil {
 		return EnrollOrbitResponse{Err: err}, nil
@@ -121,6 +124,7 @@ func (svc *Service) EnrollOrbit(ctx context.Context, hostInfo fleet.OrbitHostInf
 			"hardware_serial", hostInfo.HardwareSerial,
 			"hostname", hostInfo.Hostname,
 			"platform", hostInfo.Platform,
+			"osquery_identifier", hostInfo.OsqueryIdentifier,
 		),
 		level.Info,
 	)

server/service/orbit_client.go

@@ -185,11 +185,12 @@ func (oc *OrbitClient) Ping() error {
 func (oc *OrbitClient) enroll() (string, error) {
 	verb, path := "POST", "/api/fleet/orbit/enroll"
 	params := EnrollOrbitRequest{
-		EnrollSecret:   oc.enrollSecret,
-		HardwareUUID:   oc.hostInfo.HardwareUUID,
-		HardwareSerial: oc.hostInfo.HardwareSerial,
-		Hostname:       oc.hostInfo.Hostname,
-		Platform:       oc.hostInfo.Platform,
+		EnrollSecret:      oc.enrollSecret,
+		HardwareUUID:      oc.hostInfo.HardwareUUID,
+		HardwareSerial:    oc.hostInfo.HardwareSerial,
+		Hostname:          oc.hostInfo.Hostname,
+		Platform:          oc.hostInfo.Platform,
+		OsqueryIdentifier: oc.hostInfo.OsqueryIdentifier,
 	}
 	var resp EnrollOrbitResponse
 	err := oc.request(verb, path, params, &resp)

tools/tuf/test/create_repository.sh

@@ -29,7 +29,7 @@ SWIFT_DIALOG_MACOS_APP_VERSION=2.2.1
 SWIFT_DIALOG_MACOS_APP_BUILD_VERSION=4591
 
 if [[ -z "$OSQUERY_VERSION" ]]; then
-    OSQUERY_VERSION=5.10.1
+    OSQUERY_VERSION=5.10.2
 fi
 
 mkdir -p $TUF_PATH/tmp