Verify VPP: core implementation (#30295)

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

<!-- Note that API documentation changes are now addressed by the
product design team. -->

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.
- [x] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements)
- For database migrations:
- [x] Ensured the correct collation is explicitly set for character
columns (`COLLATE utf8mb4_unicode_ci`).
- [x] Added/updated automated tests
- [x] Manual QA for all new/changed functionality

changes/29830-verify-vpp-db-migration

@@ -0,0 +1 @@
+- Adds functionality for verifying installation of VPP apps.

cmd/fleet/cron.go

@@ -706,11 +706,16 @@ func newWorkerIntegrationsSchedule(
 		Commander:             commander,
 		BootstrapPackageStore: bootstrapPackageStore,
 	}
+	vppVerify := &worker.VPPVerification{
+		Datastore: ds,
+		Log:       logger,
+		Commander: commander,
+	}
 	dbMigrate := &worker.DBMigration{
 		Datastore: ds,
 		Log:       logger,
 	}
-	w.Register(jira, zendesk, macosSetupAsst, appleMDM, dbMigrate)
+	w.Register(jira, zendesk, macosSetupAsst, appleMDM, dbMigrate, vppVerify)
 
 	// Read app config a first time before starting, to clear up any failer client
 	// configuration if we're not on a fleet-owned server. Technically, the ServerURL

cmd/fleet/serve.go

@@ -1171,6 +1171,8 @@ the way that the Fleet server works.
 				ddmService := service.NewMDMAppleDDMService(ds, logger)
 				mdmCheckinAndCommandService := service.NewMDMAppleCheckinAndCommandService(ds, commander, logger)
 
+				mdmCheckinAndCommandService.RegisterResultsHandler("InstalledApplicationList", service.NewInstalledApplicationListResultsHandler(ds, commander, logger, config.Server.VPPVerifyTimeout, config.Server.VPPVerifyRequestDelay))
+
 				hasSCEPChallenge, err := checkMDMAssets([]fleet.MDMAssetName{fleet.MDMAssetSCEPChallenge})
 				if err != nil {
 					initFatal(err, "checking SCEP challenge in database")

pkg/mdm/mdmtest/apple.go

@@ -808,6 +808,7 @@ func (c *TestAppleMDMClient) AcknowledgeInstalledApplicationList(udid, cmdUUID s
 			"Name":         s.Name,
 			"ShortVersion": s.Version,
 			"Identifier":   s.BundleIdentifier,
+			"Installing":   false,
 		})
 	}
 

server/config/config.go

@@ -92,14 +92,16 @@ type ServerConfig struct {
 	Cert                        string
 	Key                         string
 	TLS                         bool
-	TLSProfile                  string `yaml:"tls_compatibility"`
-	URLPrefix                   string `yaml:"url_prefix"`
-	Keepalive                   bool   `yaml:"keepalive"`
-	SandboxEnabled              bool   `yaml:"sandbox_enabled"`
-	WebsocketsAllowUnsafeOrigin bool   `yaml:"websockets_allow_unsafe_origin"`
-	FrequentCleanupsEnabled     bool   `yaml:"frequent_cleanups_enabled"`
-	ForceH2C                    bool   `yaml:"force_h2c"`
-	PrivateKey                  string `yaml:"private_key"`
+	TLSProfile                  string        `yaml:"tls_compatibility"`
+	URLPrefix                   string        `yaml:"url_prefix"`
+	Keepalive                   bool          `yaml:"keepalive"`
+	SandboxEnabled              bool          `yaml:"sandbox_enabled"`
+	WebsocketsAllowUnsafeOrigin bool          `yaml:"websockets_allow_unsafe_origin"`
+	FrequentCleanupsEnabled     bool          `yaml:"frequent_cleanups_enabled"`
+	ForceH2C                    bool          `yaml:"force_h2c"`
+	PrivateKey                  string        `yaml:"private_key"`
+	VPPVerifyTimeout            time.Duration `yaml:"vpp_verify_timeout"`
+	VPPVerifyRequestDelay       time.Duration `yaml:"vpp_verify_request_delay"`
 }
 
 func (s *ServerConfig) DefaultHTTPServer(ctx context.Context, handler http.Handler) *http.Server {
@@ -1093,6 +1095,8 @@ func (man Manager) addConfigs() {
 	man.addConfigBool("server.frequent_cleanups_enabled", false, "Enable frequent cleanups of expired data (15 minute interval)")
 	man.addConfigBool("server.force_h2c", false, "Force the fleet server to use HTTP2 cleartext aka h2c (ignored if using TLS)")
 	man.addConfigString("server.private_key", "", "Used for encrypting sensitive data, such as MDM certificates.")
+	man.addConfigDuration("server.vpp_verify_timeout", 5*time.Minute, "Maximum amout of time to wait for VPP app install verification")
+	man.addConfigDuration("server.vpp_verify_request_delay", 5*time.Second, "Delay in between requests to verify VPP app installs")
 
 	// Hide the sandbox flag as we don't want it to be discoverable for users for now
 	man.hideConfig("server.sandbox_enabled")
@@ -1517,6 +1521,8 @@ func (man Manager) LoadConfig() FleetConfig {
 			FrequentCleanupsEnabled:     man.getConfigBool("server.frequent_cleanups_enabled"),
 			ForceH2C:                    man.getConfigBool("server.force_h2c"),
 			PrivateKey:                  man.getConfigString("server.private_key"),
+			VPPVerifyTimeout:            man.getConfigDuration("server.vpp_verify_timeout"),
+			VPPVerifyRequestDelay:       man.getConfigDuration("server.vpp_verify_request_delay"),
 		},
 		Auth: AuthConfig{
 			BcryptCost:               man.getConfigInt("auth.bcrypt_cost"),

server/datastore/mysql/activities.go

@@ -76,25 +76,33 @@ func (ds *Datastore) NewActivity(
 			cmdUUID = vppPtrAct.CommandUUID
 			hostID = vppPtrAct.HostID
 		}
-		// NOTE: ideally this would be called in the same transaction as storing
-		// the nanomdm command results, but the current design doesn't allow for
-		// that with the nano store being a distinct entity to our datastore (we
-		// should get rid of that distinction eventually, we've broken it already
-		// in some places and it doesn't bring much benefit anymore).
-		//
-		// Instead, this gets called from CommandAndReportResults, which is
-		// executed after the results have been saved in nano, but we already
-		// accept this non-transactional fact for many other states we manage in
-		// Fleet (wipe, lock results, setup experience results, etc. - see all
-		// critical data that gets updated in CommandAndReportResults) so there's
-		// no reason to treat the unified queue differently.
-		//
-		// This place here is a bit hacky but perfect for VPP apps as the activity
-		// gets created only when the MDM command status is in a final state
-		// (success or failure), which is exactly when we want to activate the next
-		// activity.
-		if _, err := ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), hostID, cmdUUID); err != nil {
-			return ctxerr.Wrap(ctx, err, "activate next activity from VPP app install")
+
+		activateNext := vppAct.Status != string(fleet.SoftwareInstalled)
+		if vppPtrAct != nil {
+			activateNext = vppPtrAct.Status != string(fleet.SoftwareInstalled)
+		}
+
+		if activateNext {
+			// NOTE: ideally this would be called in the same transaction as storing
+			// the nanomdm command results, but the current design doesn't allow for
+			// that with the nano store being a distinct entity to our datastore (we
+			// should get rid of that distinction eventually, we've broken it already
+			// in some places and it doesn't bring much benefit anymore).
+			//
+			// Instead, this gets called from CommandAndReportResults, which is
+			// executed after the results have been saved in nano, but we already
+			// accept this non-transactional fact for many other states we manage in
+			// Fleet (wipe, lock results, setup experience results, etc. - see all
+			// critical data that gets updated in CommandAndReportResults) so there's
+			// no reason to treat the unified queue differently.
+			//
+			// This place here is a bit hacky but perfect for VPP apps as the activity
+			// gets created only when the MDM command status is in a final state
+			// (success or failure), which is exactly when we want to activate the next
+			// activity.
+			if _, err := ds.activateNextUpcomingActivity(ctx, ds.writer(ctx), hostID, cmdUUID); err != nil {
+				return ctxerr.Wrap(ctx, err, "activate next activity from VPP app install")
+			}
 		}
 	}
 
@@ -1464,6 +1472,8 @@ WHERE
 <dict>
     <key>Command</key>
     <dict>
+		<key>InstallAsManaged</key>
+		<true/>
         <key>ManagementFlags</key>
         <integer>0</integer>
         <key>Options</key>

server/datastore/mysql/mdm.go

@@ -1967,3 +1967,23 @@ GROUP BY
 
 	return counts, nil
 }
+
+func (ds *Datastore) GetAcknowledgedMDMCommandsByHost(ctx context.Context, hostUUID, commandType string) ([]string, error) {
+	stmt := `
+SELECT
+    nvq.command_uuid AS command_uuid
+FROM
+    nano_view_queue nvq
+WHERE
+   nvq.active = 1 
+   AND nvq.id = ?
+   AND nvq.request_type = ?
+   AND status != 'Acknowledged'`
+
+	var cmdUUIDs []string
+	if err := sqlx.SelectContext(ctx, ds.reader(ctx), &cmdUUIDs, stmt, hostUUID, commandType); err != nil {
+		return nil, ctxerr.Wrap(ctx, err, "get pending mdm commands by host")
+	}
+
+	return cmdUUIDs, nil
+}

server/datastore/mysql/migrations/tables/20250624140757_VPPAppVerifyInstall.go

@@ -0,0 +1,42 @@
+package tables
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/fleetdm/fleet/v4/server/fleet"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20250624140757, Down_20250624140757)
+}
+
+func Up_20250624140757(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+	ALTER TABLE host_vpp_software_installs
+		ADD COLUMN verification_command_uuid VARCHAR(127) NULL,
+		ADD COLUMN verification_at DATETIME(6) NULL,
+		ADD COLUMN verification_failed_at DATETIME(6) NULL
+		`)
+	if err != nil {
+		return fmt.Errorf("failed to add host_vpp_software_installs.verification_command_uuid: %w", err)
+	}
+
+	_, err = tx.Exec(`
+UPDATE
+	host_vpp_software_installs hvsi
+	INNER JOIN nano_command_results ncr ON ncr.command_uuid = hvsi.command_uuid
+SET
+	hvsi.verification_at = IF(ncr.status = 'Acknowledged', CURRENT_TIMESTAMP(6), NULL),
+	hvsi.verification_failed_at = IF(ncr.status = ? OR ncr.status = ?, CURRENT_TIMESTAMP(6), NULL);
+	`, fleet.MDMAppleStatusError, fleet.MDMAppleStatusCommandFormatError)
+	if err != nil {
+		return fmt.Errorf("failed to set existing vpp install verification statuses: %w", err)
+	}
+
+	return nil
+}
+
+func Down_20250624140757(tx *sql.Tx) error {
+	return nil
+}

server/datastore/mysql/migrations/tables/20250624140757_VPPAppVerifyInstall_test.go

@@ -0,0 +1,108 @@
+package tables
+
+import (
+	"testing"
+	"time"
+
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUp_20250624140757(t *testing.T) {
+	db := applyUpToPrev(t)
+
+	// Create user
+	u1 := execNoErrLastID(t, db, `INSERT INTO users (name, email, password, salt) VALUES (?, ?, ?, ?)`, "u1", "u1@b.c", "1234", "salt")
+	// Create host
+	insertHostStmt := `
+			INSERT INTO hosts (
+				hostname, uuid, platform, osquery_version, os_version, build, platform_like, code_name,
+				cpu_type, cpu_subtype, cpu_brand, hardware_vendor, hardware_model, hardware_version,
+				hardware_serial, computer_name, team_id
+			) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+		`
+	hostName := "Dummy Hostname"
+	hostUUID := "12345678-1234-1234-1234-123456789012"
+	hostPlatform := "darwin"
+	osqueryVer := "5.9.1"
+	osVersion := "macOS 14.5"
+	buildVersion := "10.0.19042.1234"
+	platformLike := "darwin"
+	codeName := "20H2"
+	cpuType := "x86_64"
+	cpuSubtype := "x86_64"
+	cpuBrand := "Intel"
+	hwVendor := "Apple Inc."
+	hwModel := "Mac14,3"
+	hwVersion := "1.0"
+	hwSerial := "ABCDEFGHIJ"
+	computerName := "DESKTOP-TEST"
+
+	hostID := execNoErrLastID(t, db, insertHostStmt, hostName, hostUUID, hostPlatform, osqueryVer,
+		osVersion, buildVersion, platformLike, codeName, cpuType, cpuSubtype, cpuBrand, hwVendor, hwModel, hwVersion, hwSerial, computerName, nil)
+
+	// Create VPP app
+	adamID := "a"
+	execNoErr(
+		t, db, `INSERT INTO vpp_apps (adam_id, platform) VALUES (?,?)`, adamID, hostPlatform,
+	)
+
+	// Host MDM setup
+	execNoErr(t, db, `INSERT INTO nano_devices (id, authenticate) VALUES (?, ?)`, hostUUID, "auth")
+	execNoErr(t, db, `
+INSERT INTO nano_enrollments (id, device_id, type, topic, push_magic, token_hex, last_seen_at)
+VALUES (?, ?, ?, ?, ?, ?, ?)`, hostUUID, hostUUID, "device", "topic", "magic", "hex", time.Now())
+
+	insertVPPAppInstall := func(status string) int64 {
+		installedUUID := uuid.NewString()
+
+		execNoErr(t, db, `INSERT INTO nano_commands (command_uuid, request_type, command) VALUES (?, ?, ?)`,
+			installedUUID, "InstallApplication", "<?xml")
+		execNoErr(t, db, `INSERT INTO nano_enrollment_queue (id, command_uuid) VALUES (?, ?)`,
+			hostUUID, installedUUID)
+
+		execNoErr(t, db, `INSERT INTO nano_command_results (id, command_uuid, status, result) VALUES (?, ?, ?, ?)`,
+			hostUUID, installedUUID, status, "<?xml")
+
+		// create an install on a known host
+		return execNoErrLastID(t, db, `INSERT INTO host_vpp_software_installs (host_id, adam_id, command_uuid, user_id, platform) VALUES (?,?,?,?,?)`, hostID, adamID, installedUUID, u1, "darwin")
+	}
+
+	hvsi1 := insertVPPAppInstall(fleet.MDMAppleStatusAcknowledged)
+	hvsi2 := insertVPPAppInstall(fleet.MDMAppleStatusError)
+	hvsi3 := insertVPPAppInstall(fleet.MDMAppleStatusCommandFormatError)
+	hvsi4 := insertVPPAppInstall(fleet.MDMAppleStatusNotNow)
+	hvsi5 := insertVPPAppInstall(fleet.MDMAppleStatusIdle)
+
+	// Apply current migration.
+	applyNext(t, db)
+
+	// For the acknowledged command, we should mark as verified
+	var verifiedTime *time.Time
+	require.NoError(t, db.Get(&verifiedTime, `SELECT verification_at FROM host_vpp_software_installs WHERE id = ?`, hvsi1))
+	require.NotNil(t, verifiedTime)
+	require.NotZero(t, *verifiedTime)
+
+	// For the error command, we should mark as failed
+	var failedTime *time.Time
+	require.NoError(t, db.Get(&failedTime, `SELECT verification_failed_at FROM host_vpp_software_installs WHERE id = ?`, hvsi2))
+	require.NotNil(t, failedTime)
+	require.NotZero(t, *failedTime)
+
+	// For the format error command, we should mark as failed
+	require.NoError(t, db.Get(&failedTime, `SELECT verification_failed_at FROM host_vpp_software_installs WHERE id = ?`, hvsi3))
+	require.NotNil(t, failedTime)
+	require.NotZero(t, *failedTime)
+
+	// For the notnow and idle command, no status set (install hasn't finalized yet)
+	require.NoError(t, db.Get(&verifiedTime, `SELECT verification_at FROM host_vpp_software_installs WHERE id = ?`, hvsi4))
+	require.Nil(t, verifiedTime)
+	require.NoError(t, db.Get(&failedTime, `SELECT verification_failed_at FROM host_vpp_software_installs WHERE id = ?`, hvsi4))
+	require.Nil(t, failedTime)
+
+	require.NoError(t, db.Get(&verifiedTime, `SELECT verification_at FROM host_vpp_software_installs WHERE id = ?`, hvsi5))
+	require.Nil(t, verifiedTime)
+	require.NoError(t, db.Get(&failedTime, `SELECT verification_failed_at FROM host_vpp_software_installs WHERE id = ?`, hvsi5))
+	require.Nil(t, failedTime)
+}

server/datastore/mysql/software.go

@@ -3279,6 +3279,12 @@ func (ds *Datastore) ListHostSoftware(ctx context.Context, host *fleet.Host, opt
 			s.Version = installedTitle.Version
 			s.BundleIdentifier = installedTitle.BundleIdentifier
 		}
+		if s.VPPAppAdamID != nil {
+			// Override the status; if there's a pending re-install, we should show that status.
+			if hs, ok := byVPPAdamID[*s.VPPAppAdamID]; ok {
+				s.Status = hs.Status
+			}
+		}
 		hostVPPInstalledTitles[s.ID] = s
 	}
 

server/datastore/mysql/vpp.go

@@ -1723,3 +1723,96 @@ FROM vpp_apps`
 
 	return apps, nil
 }
+
+func (ds *Datastore) GetVPPInstallsByVerificationUUID(ctx context.Context, verificationUUID string) ([]*fleet.HostVPPSoftwareInstall, error) {
+	stmt := `
+SELECT
+	hvsi.command_uuid AS command_uuid,
+	hvsi.host_id AS host_id,
+	ncr.updated_at AS ack_at,
+	ncr.status AS install_command_status,
+	va.bundle_identifier AS bundle_identifier
+FROM nano_command_results ncr
+JOIN host_vpp_software_installs hvsi ON hvsi.command_uuid = ncr.command_uuid
+JOIN vpp_apps va ON va.adam_id = hvsi.adam_id AND va.platform = hvsi.platform
+WHERE hvsi.verification_command_uuid = ?
+	`
+
+	var result []*fleet.HostVPPSoftwareInstall
+	if err := sqlx.SelectContext(ctx, ds.reader(ctx), &result, stmt, verificationUUID); err != nil {
+		if errors.Is(err, sql.ErrNoRows) {
+			return nil, notFound("HostVPPSoftwareInstall")
+		}
+		return nil, ctxerr.Wrap(ctx, err, "get vpp install ack time by verification uuid")
+	}
+
+	return result, nil
+}
+
+func (ds *Datastore) UpdateVPPInstallVerificationCommand(ctx context.Context, installUUID, verifyCommandUUID string) error {
+	stmt := `
+UPDATE host_vpp_software_installs
+SET verification_command_uuid = ? 
+WHERE command_uuid = ?
+	`
+
+	if _, err := ds.writer(ctx).ExecContext(ctx, stmt, verifyCommandUUID, installUUID); err != nil {
+		return ctxerr.Wrap(ctx, err, "update vpp install verification command")
+	}
+
+	return nil
+}
+
+func (ds *Datastore) UpdateVPPInstallVerificationCommandByVerifyUUID(ctx context.Context, oldVerifyUUID, verifyCommandUUID string) error {
+	stmt := `
+UPDATE host_vpp_software_installs
+SET verification_command_uuid = ? 
+WHERE verification_command_uuid = ?
+	`
+
+	if _, err := ds.writer(ctx).ExecContext(ctx, stmt, verifyCommandUUID, oldVerifyUUID); err != nil {
+		return ctxerr.Wrap(ctx, err, "update vpp install verification command")
+	}
+
+	return nil
+}
+
+func (ds *Datastore) SetVPPInstallAsVerified(ctx context.Context, hostID uint, installUUID string) error {
+	stmt := `
+UPDATE host_vpp_software_installs
+SET verification_at = CURRENT_TIMESTAMP(6)
+WHERE command_uuid = ?
+	`
+
+	return ds.withTx(ctx, func(tx sqlx.ExtContext) error {
+		if _, err := tx.ExecContext(ctx, stmt, installUUID); err != nil {
+			return ctxerr.Wrap(ctx, err, "set vpp install as verified")
+		}
+
+		if _, err := ds.activateNextUpcomingActivity(ctx, tx, hostID, installUUID); err != nil {
+			return ctxerr.Wrap(ctx, err, "activate next activity from VPP app install verify")
+		}
+
+		return nil
+	})
+}
+
+func (ds *Datastore) SetVPPInstallAsFailed(ctx context.Context, hostID uint, installUUID string) error {
+	stmt := `
+UPDATE host_vpp_software_installs
+SET verification_failed_at = CURRENT_TIMESTAMP(6)
+WHERE command_uuid = ?
+	`
+
+	return ds.withTx(ctx, func(tx sqlx.ExtContext) error {
+		if _, err := tx.ExecContext(ctx, stmt, installUUID); err != nil {
+			return ctxerr.Wrap(ctx, err, "set vpp install as failed")
+		}
+
+		if _, err := ds.activateNextUpcomingActivity(ctx, tx, hostID, installUUID); err != nil {
+			return ctxerr.Wrap(ctx, err, "activate next activity from VPP app install failed")
+		}
+
+		return nil
+	})
+}

server/fleet/datastore.go

@@ -651,6 +651,24 @@ type Datastore interface {
 	// from the title IDs to the categories assigned to the installers for those titles.
 	GetCategoriesForSoftwareTitles(ctx context.Context, softwareTitleIDs []uint, team_id *uint) (map[uint][]string, error)
 
+	// UpdateVPPInstallVerificationCommand updates the verification command UUID associated with the
+	// given install attempt (InstallApplication command)
+	UpdateVPPInstallVerificationCommand(ctx context.Context, installUUID, verifyCommandUUID string) error
+	// SetVPPInstallAsVerified marks the VPP app install attempt as "verified" (Fleet has validated
+	// that it's installed on the device).
+	SetVPPInstallAsVerified(ctx context.Context, hostID uint, installUUID string) error
+	// UpdateVPPInstallVerificationCommandByVerifyUUID updates the verification command UUID for all
+	// VPP app install attempts were related to oldVerifyUUID.
+	UpdateVPPInstallVerificationCommandByVerifyUUID(ctx context.Context, oldVerifyUUID, verifyCommandUUID string) error
+	// GetAcknowledgedMDMCommandsByHost gets all commands of the given type that are in the
+	// "Acknowledged" state.
+	GetAcknowledgedMDMCommandsByHost(ctx context.Context, hostUUID, commandType string) ([]string, error)
+	// GetVPPInstallsByVerificationUUID gets a HostVPPSoftwareInstall by verification command UUID.
+	GetVPPInstallsByVerificationUUID(ctx context.Context, verificationUUID string) ([]*HostVPPSoftwareInstall, error)
+	// SetVPPInstallAsFailed marks a VPP app install attempt as failed (Fleet couldn't validate that
+	// it was installed on the host).
+	SetVPPInstallAsFailed(ctx context.Context, hostID uint, installUUID string) error
+
 	///////////////////////////////////////////////////////////////////////////////
 	// OperatingSystemsStore
 

server/fleet/mdm.go

@@ -844,10 +844,11 @@ func FilterMacOSOnlyProfilesFromIOSIPadOS(profiles []*MDMAppleProfilePayload) []
 
 // RefetchBaseCommandUUIDPrefix and below command prefixes are the prefixes used for MDM commands used to refetch information from iOS/iPadOS devices.
 const (
-	RefetchBaseCommandUUIDPrefix   = "REFETCH-"
-	RefetchDeviceCommandUUIDPrefix = RefetchBaseCommandUUIDPrefix + "DEVICE-"
-	RefetchAppsCommandUUIDPrefix   = RefetchBaseCommandUUIDPrefix + "APPS-"
-	RefetchCertsCommandUUIDPrefix  = RefetchBaseCommandUUIDPrefix + "CERTS-"
+	RefetchBaseCommandUUIDPrefix           = "REFETCH-"
+	RefetchDeviceCommandUUIDPrefix         = RefetchBaseCommandUUIDPrefix + "DEVICE-"
+	RefetchAppsCommandUUIDPrefix           = RefetchBaseCommandUUIDPrefix + "APPS-"
+	RefetchCertsCommandUUIDPrefix          = RefetchBaseCommandUUIDPrefix + "CERTS-"
+	RefetchVPPAppInstallsCommandUUIDPrefix = RefetchBaseCommandUUIDPrefix + "VPP-INSTALLS-"
 )
 
 // VPPTokenInfo is the representation of the VPP token that we send out via API.
@@ -1017,3 +1018,14 @@ type MDMConfigProfileStatus struct {
 type MDMWipeMetadata struct {
 	Windows *MDMWindowsWipeMetadata
 }
+
+type MDMCommandResults interface {
+	// Raw returns the raw bytes of the MDM command result XML.
+	Raw() []byte
+	// UUID returns the UUID of the command that returned these results.
+	UUID() string
+	// HostUUID returns the UUID of the host that ran the command and returned these results.
+	HostUUID() string
+}
+
+type CommandHandler func(ctx context.Context, results MDMCommandResults) error

server/fleet/software.go

@@ -91,6 +91,9 @@ type Software struct {
 	NameSource string `json:"-" db:"name_source"`
 	// Checksum is the unique checksum generated for this Software.
 	Checksum string `json:"-" db:"checksum"`
+	// TODO: should we create a separate type? Feels like this field shouldn't be here since it's
+	// just used for VPP install verification.
+	Installed bool `json:"-"`
 }
 
 func (Software) AuthzType() string {

server/fleet/vpp.go

@@ -122,3 +122,11 @@ type ErrVPPTokenTeamConstraint struct {
 func (e ErrVPPTokenTeamConstraint) Error() string {
 	return fmt.Sprintf("Error: %q team already has a VPP token. Each team can only have one VPP token.", e.Name)
 }
+
+type HostVPPSoftwareInstall struct {
+	InstallCommandUUID   string     `db:"command_uuid"`
+	InstallCommandAckAt  *time.Time `db:"ack_at"`
+	HostID               uint       `db:"host_id"`
+	InstallCommandStatus string     `db:"install_command_status"`
+	BundleIdentifier     string     `db:"bundle_identifier"`
+}

server/mdm/apple/apple_mdm.go

@@ -1234,7 +1234,7 @@ func IOSiPadOSRefetch(ctx context.Context, ds fleet.Datastore, commander *MDMApp
 		}
 	}
 	if len(installedAppsUUIDs) > 0 {
-		err = commander.InstalledApplicationList(ctx, installedAppsUUIDs, fleet.RefetchAppsCommandUUIDPrefix+commandUUID)
+		err = commander.InstalledApplicationList(ctx, installedAppsUUIDs, fleet.RefetchAppsCommandUUIDPrefix+commandUUID, false)
 		if err != nil {
 			return ctxerr.Wrap(ctx, err, "send InstalledApplicationList commands to ios and ipados devices")
 		}

server/mdm/apple/commander.go

@@ -310,7 +310,7 @@ func (svc *MDMAppleCommander) DeviceInformation(ctx context.Context, hostUUIDs [
 	return svc.EnqueueCommand(ctx, hostUUIDs, raw)
 }
 
-func (svc *MDMAppleCommander) InstalledApplicationList(ctx context.Context, hostUUIDs []string, cmdUUID string) error {
+func (svc *MDMAppleCommander) InstalledApplicationList(ctx context.Context, hostUUIDs []string, cmdUUID string, managedOnly bool) error {
 	raw := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
@@ -318,7 +318,7 @@ func (svc *MDMAppleCommander) InstalledApplicationList(ctx context.Context, host
         <key>Command</key>
         <dict>
             <key>ManagedAppsOnly</key>
-            <false/>
+            <%t/>
             <key>RequestType</key>
             <string>InstalledApplicationList</string>
             <key>Items</key>
@@ -331,7 +331,7 @@ func (svc *MDMAppleCommander) InstalledApplicationList(ctx context.Context, host
         <key>CommandUUID</key>
         <string>%s</string>
     </dict>
-</plist>`, cmdUUID)
+</plist>`, managedOnly, cmdUUID)
 
 	return svc.EnqueueCommand(ctx, hostUUIDs, raw)
 }

server/mock/datastore_mock.go

@@ -486,6 +486,18 @@ type GetSoftwareCategoryIDsFunc func(ctx context.Context, names []string) ([]uin
 
 type GetCategoriesForSoftwareTitlesFunc func(ctx context.Context, softwareTitleIDs []uint, team_id *uint) (map[uint][]string, error)
 
+type UpdateVPPInstallVerificationCommandFunc func(ctx context.Context, installUUID string, verifyCommandUUID string) error
+
+type SetVPPInstallAsVerifiedFunc func(ctx context.Context, hostID uint, installUUID string) error
+
+type UpdateVPPInstallVerificationCommandByVerifyUUIDFunc func(ctx context.Context, oldVerifyUUID string, verifyCommandUUID string) error
+
+type GetAcknowledgedMDMCommandsByHostFunc func(ctx context.Context, hostUUID string, commandType string) ([]string, error)
+
+type GetVPPInstallsByVerificationUUIDFunc func(ctx context.Context, verificationUUID string) ([]*fleet.HostVPPSoftwareInstall, error)
+
+type SetVPPInstallAsFailedFunc func(ctx context.Context, hostID uint, installUUID string) error
+
 type GetHostOperatingSystemFunc func(ctx context.Context, hostID uint) (*fleet.OperatingSystem, error)
 
 type ListOperatingSystemsFunc func(ctx context.Context) ([]fleet.OperatingSystem, error)
@@ -2091,6 +2103,24 @@ type DataStore struct {
 	GetCategoriesForSoftwareTitlesFunc        GetCategoriesForSoftwareTitlesFunc
 	GetCategoriesForSoftwareTitlesFuncInvoked bool
 
+	UpdateVPPInstallVerificationCommandFunc        UpdateVPPInstallVerificationCommandFunc
+	UpdateVPPInstallVerificationCommandFuncInvoked bool
+
+	SetVPPInstallAsVerifiedFunc        SetVPPInstallAsVerifiedFunc
+	SetVPPInstallAsVerifiedFuncInvoked bool
+
+	UpdateVPPInstallVerificationCommandByVerifyUUIDFunc        UpdateVPPInstallVerificationCommandByVerifyUUIDFunc
+	UpdateVPPInstallVerificationCommandByVerifyUUIDFuncInvoked bool
+
+	GetAcknowledgedMDMCommandsByHostFunc        GetAcknowledgedMDMCommandsByHostFunc
+	GetAcknowledgedMDMCommandsByHostFuncInvoked bool
+
+	GetVPPInstallsByVerificationUUIDFunc        GetVPPInstallsByVerificationUUIDFunc
+	GetVPPInstallsByVerificationUUIDFuncInvoked bool
+
+	SetVPPInstallAsFailedFunc        SetVPPInstallAsFailedFunc
+	SetVPPInstallAsFailedFuncInvoked bool
+
 	GetHostOperatingSystemFunc        GetHostOperatingSystemFunc
 	GetHostOperatingSystemFuncInvoked bool
 
@@ -5080,6 +5110,48 @@ func (s *DataStore) GetCategoriesForSoftwareTitles(ctx context.Context, software
 	return s.GetCategoriesForSoftwareTitlesFunc(ctx, softwareTitleIDs, team_id)
 }
 
+func (s *DataStore) UpdateVPPInstallVerificationCommand(ctx context.Context, installUUID string, verifyCommandUUID string) error {
+	s.mu.Lock()
+	s.UpdateVPPInstallVerificationCommandFuncInvoked = true
+	s.mu.Unlock()
+	return s.UpdateVPPInstallVerificationCommandFunc(ctx, installUUID, verifyCommandUUID)
+}
+
+func (s *DataStore) SetVPPInstallAsVerified(ctx context.Context, hostID uint, installUUID string) error {
+	s.mu.Lock()
+	s.SetVPPInstallAsVerifiedFuncInvoked = true
+	s.mu.Unlock()
+	return s.SetVPPInstallAsVerifiedFunc(ctx, hostID, installUUID)
+}
+
+func (s *DataStore) UpdateVPPInstallVerificationCommandByVerifyUUID(ctx context.Context, oldVerifyUUID string, verifyCommandUUID string) error {
+	s.mu.Lock()
+	s.UpdateVPPInstallVerificationCommandByVerifyUUIDFuncInvoked = true
+	s.mu.Unlock()
+	return s.UpdateVPPInstallVerificationCommandByVerifyUUIDFunc(ctx, oldVerifyUUID, verifyCommandUUID)
+}
+
+func (s *DataStore) GetAcknowledgedMDMCommandsByHost(ctx context.Context, hostUUID string, commandType string) ([]string, error) {
+	s.mu.Lock()
+	s.GetAcknowledgedMDMCommandsByHostFuncInvoked = true
+	s.mu.Unlock()
+	return s.GetAcknowledgedMDMCommandsByHostFunc(ctx, hostUUID, commandType)
+}
+
+func (s *DataStore) GetVPPInstallsByVerificationUUID(ctx context.Context, verificationUUID string) ([]*fleet.HostVPPSoftwareInstall, error) {
+	s.mu.Lock()
+	s.GetVPPInstallsByVerificationUUIDFuncInvoked = true
+	s.mu.Unlock()
+	return s.GetVPPInstallsByVerificationUUIDFunc(ctx, verificationUUID)
+}
+
+func (s *DataStore) SetVPPInstallAsFailed(ctx context.Context, hostID uint, installUUID string) error {
+	s.mu.Lock()
+	s.SetVPPInstallAsFailedFuncInvoked = true
+	s.mu.Unlock()
+	return s.SetVPPInstallAsFailedFunc(ctx, hostID, installUUID)
+}
+
 func (s *DataStore) GetHostOperatingSystem(ctx context.Context, hostID uint) (*fleet.OperatingSystem, error) {
 	s.mu.Lock()
 	s.GetHostOperatingSystemFuncInvoked = true

server/service/apple_mdm.go

@@ -53,6 +53,7 @@ import (
 	"github.com/fleetdm/fleet/v4/server/ptr"
 	"github.com/fleetdm/fleet/v4/server/service/middleware/endpoint_utils"
 	"github.com/fleetdm/fleet/v4/server/sso"
+	"github.com/fleetdm/fleet/v4/server/worker"
 	kitlog "github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/google/uuid"
@@ -3153,22 +3154,28 @@ func (svc *Service) MDMAppleDisableFileVaultAndEscrow(ctx context.Context, teamI
 ////////////////////////////////////////////////////////////////////////////////
 
 type MDMAppleCheckinAndCommandService struct {
-	ds           fleet.Datastore
-	logger       kitlog.Logger
-	commander    *apple_mdm.MDMAppleCommander
-	mdmLifecycle *mdmlifecycle.HostLifecycle
+	ds              fleet.Datastore
+	logger          kitlog.Logger
+	commander       *apple_mdm.MDMAppleCommander
+	mdmLifecycle    *mdmlifecycle.HostLifecycle
+	commandHandlers map[string][]fleet.CommandHandler
 }
 
 func NewMDMAppleCheckinAndCommandService(ds fleet.Datastore, commander *apple_mdm.MDMAppleCommander, logger kitlog.Logger) *MDMAppleCheckinAndCommandService {
 	mdmLifecycle := mdmlifecycle.New(ds, logger)
 	return &MDMAppleCheckinAndCommandService{
-		ds:           ds,
-		commander:    commander,
-		logger:       logger,
-		mdmLifecycle: mdmLifecycle,
+		ds:              ds,
+		commander:       commander,
+		logger:          logger,
+		mdmLifecycle:    mdmLifecycle,
+		commandHandlers: map[string][]fleet.CommandHandler{},
 	}
 }
 
+func (svc *MDMAppleCheckinAndCommandService) RegisterResultsHandler(commandType string, handler fleet.CommandHandler) {
+	svc.commandHandlers[commandType] = append(svc.commandHandlers[commandType], handler)
+}
+
 // Authenticate handles MDM [Authenticate][1] requests.
 //
 // This method is executed after the request has been handled by nanomdm, note
@@ -3437,7 +3444,7 @@ func (svc *MDMAppleCheckinAndCommandService) CommandAndReportResults(r *mdm.Requ
 
 	// Check if this is a result of a "refetch" command sent to iPhones/iPads
 	// to fetch their device information periodically.
-	if strings.HasPrefix(cmdResult.CommandUUID, fleet.RefetchBaseCommandUUIDPrefix) {
+	if strings.HasPrefix(cmdResult.CommandUUID, fleet.RefetchBaseCommandUUIDPrefix) && !strings.HasPrefix(cmdResult.CommandUUID, fleet.RefetchVPPAppInstallsCommandUUIDPrefix) {
 		return svc.handleRefetch(r, cmdResult)
 	}
 
@@ -3497,8 +3504,7 @@ func (svc *MDMAppleCheckinAndCommandService) CommandAndReportResults(r *mdm.Requ
 		}
 
 		// create an activity for installing only if we're in a terminal state
-		if cmdResult.Status == fleet.MDMAppleStatusAcknowledged ||
-			cmdResult.Status == fleet.MDMAppleStatusError ||
+		if cmdResult.Status == fleet.MDMAppleStatusError ||
 			cmdResult.Status == fleet.MDMAppleStatusCommandFormatError {
 			user, act, err := svc.ds.GetPastActivityDataForVPPAppInstall(r.Context, cmdResult)
 			if err != nil {
@@ -3514,10 +3520,43 @@ func (svc *MDMAppleCheckinAndCommandService) CommandAndReportResults(r *mdm.Requ
 				return nil, ctxerr.Wrap(r.Context, err, "creating activity for installed app store app")
 			}
 		}
+
+		if cmdResult.Status == fleet.MDMAppleStatusAcknowledged {
+			// Only send a new InstalledApplicationList command if there's not one in flight
+			ackCmds, err := svc.ds.GetAcknowledgedMDMCommandsByHost(r.Context, cmdResult.UDID, "InstalledApplicationList")
+			if err != nil {
+				return nil, ctxerr.Wrap(r.Context, err, "get pending mdm commands by host")
+			}
+			if len(ackCmds) == 0 {
+				cmdUUID := uuid.NewString()
+				cmdUUID = fleet.RefetchVPPAppInstallsCommandUUIDPrefix + cmdUUID
+				if err := svc.commander.InstalledApplicationList(r.Context, []string{cmdResult.UDID}, cmdUUID, true); err != nil {
+					return nil, ctxerr.Wrap(r.Context, err, "sending list app command to verify install")
+				}
+
+				// update the install record
+				if err := svc.ds.UpdateVPPInstallVerificationCommand(r.Context, cmdResult.CommandUUID, cmdUUID); err != nil {
+					return nil, ctxerr.Wrap(r.Context, err, "update install record")
+				}
+
+			}
+		}
 	case "DeviceConfigured":
 		if err := svc.ds.SetHostAwaitingConfiguration(r.Context, r.ID, false); err != nil {
 			return nil, ctxerr.Wrap(r.Context, err, "failed to mark host as non longer awaiting configuration")
 		}
+	case "InstalledApplicationList":
+		level.Debug(svc.logger).Log("msg", "calling handlers for InstalledApplicationList")
+		res, err := NewInstalledApplicationListResult(r.Context, cmdResult.Raw, cmdResult.CommandUUID, cmdResult.UDID)
+		if err != nil {
+			return nil, ctxerr.Wrap(r.Context, err, "new installed application list result")
+		}
+
+		for _, f := range svc.commandHandlers["InstalledApplicationList"] {
+			if err := f(r.Context, res); err != nil {
+				return nil, ctxerr.Wrap(r.Context, err, "InstalledApplicationList handler failed")
+			}
+		}
 	}
 
 	return nil, nil
@@ -3689,6 +3728,139 @@ func (svc *MDMAppleCheckinAndCommandService) handleRefetchDeviceResults(ctx cont
 	return nil, nil
 }
 
+type InstalledApplicationListResult interface {
+	fleet.MDMCommandResults
+	AvailableApps() []fleet.Software
+}
+
+type installedApplicationListResult struct {
+	raw           []byte
+	availableApps []fleet.Software
+	uuid          string
+	hostUUID      string
+}
+
+func (i *installedApplicationListResult) Raw() []byte                     { return i.raw }
+func (i *installedApplicationListResult) UUID() string                    { return i.uuid }
+func (i *installedApplicationListResult) HostUUID() string                { return i.hostUUID }
+func (i *installedApplicationListResult) AvailableApps() []fleet.Software { return i.availableApps }
+
+func NewInstalledApplicationListResult(ctx context.Context, rawResult []byte, uuid, hostUUID string) (InstalledApplicationListResult, error) {
+	list, err := unmarshalAppList(ctx, rawResult, "apps")
+	if err != nil {
+		return nil, ctxerr.Wrap(ctx, err, "unmarshal app list for new installed application list result")
+	}
+
+	return &installedApplicationListResult{
+		raw:           rawResult,
+		uuid:          uuid,
+		availableApps: list,
+		hostUUID:      hostUUID,
+	}, nil
+}
+
+func NewInstalledApplicationListResultsHandler(
+	ds fleet.Datastore,
+	commander *apple_mdm.MDMAppleCommander,
+	logger kitlog.Logger,
+	verifyTimeout, verifyRequestDelay time.Duration,
+) func(ctx context.Context, commandResults fleet.MDMCommandResults) error {
+	return func(ctx context.Context, commandResults fleet.MDMCommandResults) error {
+		installedAppResult, ok := commandResults.(InstalledApplicationListResult)
+		if !ok {
+			return ctxerr.New(ctx, "unexpected results type")
+		}
+
+		// Then it's not a command sent by Fleet, so skip it
+		if !strings.HasPrefix(installedAppResult.UUID(), fleet.RefetchVPPAppInstallsCommandUUIDPrefix) {
+			return nil
+		}
+
+		installedApps := installedAppResult.AvailableApps()
+
+		if len(installedApps) == 0 {
+			// Nothing to do
+			return nil
+		}
+
+		// Get installs that should be verified by this InstalledApplicationList command
+		installs, err := ds.GetVPPInstallsByVerificationUUID(ctx, installedAppResult.UUID())
+		if err != nil {
+			return ctxerr.Wrap(ctx, err, "InstalledApplicationList handler: getting install record")
+		}
+
+		installsByBundleID := map[string]*fleet.HostVPPSoftwareInstall{}
+		for _, install := range installs {
+			installsByBundleID[install.BundleIdentifier] = install
+		}
+
+		var poll bool
+		for _, a := range installedApps {
+			install, ok := installsByBundleID[a.BundleIdentifier]
+			if !ok {
+				continue
+			}
+
+			var terminal bool
+			switch {
+			case a.Installed:
+				if err := ds.SetVPPInstallAsVerified(ctx, install.HostID, install.InstallCommandUUID); err != nil {
+					return ctxerr.Wrap(ctx, err, "InstalledApplicationList handler: set vpp install verified")
+				}
+
+				terminal = true
+			case install.InstallCommandAckAt != nil && time.Since(*install.InstallCommandAckAt) > verifyTimeout:
+				if err := ds.SetVPPInstallAsFailed(ctx, install.HostID, install.InstallCommandUUID); err != nil {
+					return ctxerr.Wrap(ctx, err, "InstalledApplicationList handler: set vpp install failed")
+				}
+
+				terminal = true
+			}
+
+			if !terminal {
+				poll = true
+				continue
+			}
+
+			// this might be a setup experience VPP install, so we'll try to update setup experience status
+			if updated, err := maybeUpdateSetupExperienceStatus(ctx, ds, fleet.SetupExperienceVPPInstallResult{
+				HostUUID:      installedAppResult.HostUUID(),
+				CommandUUID:   install.InstallCommandUUID,
+				CommandStatus: install.InstallCommandStatus,
+			}, true); err != nil {
+				return ctxerr.Wrap(ctx, err, "updating setup experience status from VPP install result")
+			} else if updated {
+				level.Debug(logger).Log("msg", "setup experience script result updated", "host_uuid", installedAppResult.HostUUID(), "execution_id", install.InstallCommandUUID)
+			}
+
+			// create an activity for installing only if we're in a terminal state
+			user, act, err := ds.GetPastActivityDataForVPPAppInstall(ctx, &mdm.CommandResults{CommandUUID: install.InstallCommandUUID, Status: install.InstallCommandStatus})
+			if err != nil {
+				if fleet.IsNotFound(err) {
+					// Then this isn't a VPP install, so no activity generated
+					return nil
+				}
+
+				return ctxerr.Wrap(ctx, err, "fetching data for installed app store app activity")
+			}
+
+			if err := newActivity(ctx, user, act, ds, logger); err != nil {
+				return ctxerr.Wrap(ctx, err, "creating activity for installed app store app")
+			}
+
+		}
+
+		if poll {
+			err := worker.QueueVPPInstallVerificationJob(ctx, ds, logger, worker.VerifyVPPTask, verifyRequestDelay, installedAppResult.HostUUID(), installedAppResult.UUID())
+			if err != nil {
+				return ctxerr.Wrap(ctx, err, "InstalledApplicationList handler: queueing vpp install verification job")
+			}
+		}
+
+		return nil
+	}
+}
+
 func unmarshalAppList(ctx context.Context, response []byte, source string) ([]fleet.Software,
 	error,
 ) {
@@ -3713,12 +3885,21 @@ func unmarshalAppList(ctx context.Context, response []byte, source string) ([]fl
 
 	var software []fleet.Software
 	for _, app := range appsResponse.InstalledApplicationList {
-		software = append(software, fleet.Software{
+		sw := fleet.Software{
 			Name:             truncateString(app["Name"], fleet.SoftwareNameMaxLength),
 			Version:          truncateString(app["ShortVersion"], fleet.SoftwareVersionMaxLength),
 			BundleIdentifier: truncateString(app["Identifier"], fleet.SoftwareBundleIdentifierMaxLength),
 			Source:           source,
-		})
+		}
+		if val, ok := app["Installing"]; ok {
+			installing, ok := val.(bool)
+			if !ok {
+				return nil, ctxerr.New(ctx, "parsing Installing key")
+			}
+
+			sw.Installed = !installing
+		}
+		software = append(software, sw)
 	}
 
 	return software, nil

server/service/apple_mdm_test.go

@@ -4226,6 +4226,8 @@ func TestUnmarshalAppList(t *testing.T) {
 			<string>com.evernote.iPhone.Evernote</string>
 			<key>Name</key>
 			<string>Evernote</string>
+			<key>Installing</key>
+			<false/>
 			<key>ShortVersion</key>
 			<string>10.98.0</string>
 		</dict>
@@ -4256,6 +4258,7 @@ func TestUnmarshalAppList(t *testing.T) {
 			Version:          "10.98.0",
 			Source:           "ios_apps",
 			BundleIdentifier: "com.evernote.iPhone.Evernote",
+			Installed:        true,
 		},
 		{
 			Name:             "Netflix",

server/service/hosts.go

@@ -1107,7 +1107,7 @@ func (svc *Service) RefetchHost(ctx context.Context, id uint) error {
 		hostMDMCommands := make([]fleet.HostMDMCommand, 0, 3)
 		cmdUUID := uuid.NewString()
 		if doAppRefetch {
-			err = svc.mdmAppleCommander.InstalledApplicationList(ctx, []string{host.UUID}, fleet.RefetchAppsCommandUUIDPrefix+cmdUUID)
+			err = svc.mdmAppleCommander.InstalledApplicationList(ctx, []string{host.UUID}, fleet.RefetchAppsCommandUUIDPrefix+cmdUUID, false)
 			if err != nil {
 				return ctxerr.Wrap(ctx, err, "refetch apps with MDM")
 			}

server/service/integration_mdm_test.go

@@ -195,9 +195,15 @@ func (s *integrationMDMTestSuite) SetupSuite() {
 		Log:       wlog,
 		Commander: mdmCommander,
 	}
+	vppVerifyJob := &worker.VPPVerification{
+		Datastore: s.ds,
+		Log:       wlog,
+		Commander: mdmCommander,
+	}
 	workr := worker.NewWorker(s.ds, wlog)
 	workr.TestIgnoreUnknownJobs = true
-	workr.Register(macosJob, appleMDMJob)
+	workr.Register(macosJob, appleMDMJob, vppVerifyJob)
+
 	s.worker = workr
 
 	// clear the jobs queue of any pending jobs generated via DB migrations
@@ -11587,6 +11593,24 @@ func (s *integrationMDMTestSuite) TestRefetchIOSIPadOS() {
 	require.Len(t, listCmdResp.Results, commandsSent)
 }
 
+func checkInstallFleetdCommandSent(t *testing.T, mdmDevice *mdmtest.TestAppleMDMClient, wantCommand bool) {
+	foundInstallFleetdCommand := false
+	cmd, err := mdmDevice.Idle()
+	require.NoError(t, err)
+	for cmd != nil {
+		var fullCmd micromdm.CommandPayload
+		require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd))
+		if manifest := fullCmd.Command.InstallEnterpriseApplication.ManifestURL; manifest != nil {
+			foundInstallFleetdCommand = true
+			require.Equal(t, "InstallEnterpriseApplication", cmd.Command.RequestType)
+			require.Contains(t, *fullCmd.Command.InstallEnterpriseApplication.ManifestURL, fleetdbase.GetPKGManifestURL())
+		}
+		cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID)
+		require.NoError(t, err)
+	}
+	require.Equal(t, wantCommand, foundInstallFleetdCommand)
+}
+
 func (s *integrationMDMTestSuite) TestVPPApps() {
 	t := s.T()
 	s.setSkipWorkerJobs(t)
@@ -12201,8 +12225,12 @@ func (s *integrationMDMTestSuite) TestVPPApps() {
 	orbitHost := createOrbitEnrolledHost(t, "darwin", "nonmdm", s.ds)
 	mdmHost, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t)
 	setOrbitEnrollment(t, mdmHost, s.ds)
+	s.runWorker()
+	checkInstallFleetdCommandSent(t, mdmDevice, true)
 	selfServiceHost, selfServiceDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t)
 	setOrbitEnrollment(t, selfServiceHost, s.ds)
+	s.runWorker()
+	checkInstallFleetdCommandSent(t, selfServiceDevice, true)
 	selfServiceToken := "selfservicetoken"
 	updateDeviceTokenForHost(t, s.ds, selfServiceHost.ID, selfServiceToken)
 	s.appleVPPConfigSrvConfig.SerialNumbers = append(s.appleVPPConfigSrvConfig.SerialNumbers, selfServiceDevice.SerialNumber)
@@ -12383,8 +12411,10 @@ func (s *integrationMDMTestSuite) TestVPPApps() {
 		fmt.Sprint(team.ID), "software_title_id", fmt.Sprint(macOSTitleID))
 	require.Equal(t, 1, countResp.Count)
 
+	s.runWorker()
+
 	// Simulate successful installation on the host
-	var cmdUUID string
+	var installCmdUUID string
 	cmd, err = mdmDevice.Idle()
 	require.NoError(t, err)
 	for cmd != nil {
@@ -12392,15 +12422,40 @@ func (s *integrationMDMTestSuite) TestVPPApps() {
 		switch cmd.Command.RequestType { //nolint:gocritic // ignore singleCaseSwitch
 		case "InstallApplication":
 			require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd))
-			cmdUUID = cmd.CommandUUID
+			installCmdUUID = cmd.CommandUUID
 			cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID)
 			require.NoError(t, err)
 		}
 	}
 
+	s.runWorker()
+
+	cmd, err = mdmDevice.Idle()
+	require.NoError(t, err)
+	for cmd != nil {
+		var fullCmd micromdm.CommandPayload
+		switch cmd.Command.RequestType { //nolint:gocritic // ignore singleCaseSwitch
+		case "InstalledApplicationList":
+			require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd))
+			cmd, err = mdmDevice.AcknowledgeInstalledApplicationList(mdmDevice.UUID, cmd.CommandUUID, []fleet.Software{{Name: addedApp.Name, BundleIdentifier: addedApp.BundleIdentifier, Version: addedApp.LatestVersion}})
+			require.NoError(t, err)
+		default:
+			require.Fail(t, "unexpected command type", cmd.Command.RequestType)
+		}
+	}
+
 	listResp = listHostsResponse{}
 	s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "installed", "team_id",
 		fmt.Sprint(team.ID), "software_title_id", fmt.Sprint(macOSTitleID))
+	mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error {
+		var types []string
+		err := sqlx.SelectContext(context.Background(), q, &types, "SELECT activity_type FROM upcoming_activities WHERE host_id = ?", mdmHost.ID)
+		require.NoError(t, err)
+
+		require.Empty(t, types)
+
+		return nil
+	})
 	require.Len(t, listResp.Hosts, 1)
 	countResp = countHostsResponse{}
 	s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "installed", "team_id",
@@ -12415,7 +12470,7 @@ func (s *integrationMDMTestSuite) TestVPPApps() {
 			mdmHost.DisplayName(),
 			addedApp.Name,
 			addedApp.AdamID,
-			cmdUUID,
+			installCmdUUID,
 			fleet.SoftwareInstalled,
 		),
 		0,
@@ -12436,7 +12491,7 @@ func (s *integrationMDMTestSuite) TestVPPApps() {
 	require.Equal(t, got1.AppStoreApp.Version, addedApp.LatestVersion)
 	require.NotNil(t, got1.Status)
 	require.Equal(t, *got1.Status, fleet.SoftwareInstalled)
-	require.Equal(t, got1.AppStoreApp.LastInstall.CommandUUID, cmdUUID)
+	require.Equal(t, got1.AppStoreApp.LastInstall.CommandUUID, installCmdUUID)
 	require.NotNil(t, got1.AppStoreApp.LastInstall.InstalledAt)
 	require.Equal(t, got2.Name, "App 2")
 	require.NotNil(t, got2.Status)
@@ -12463,7 +12518,7 @@ func (s *integrationMDMTestSuite) TestVPPApps() {
 	require.Equal(t, got1.AppStoreApp.Version, addedApp.LatestVersion)
 	require.NotNil(t, got1.Status)
 	require.Equal(t, *got1.Status, fleet.SoftwareInstalled)
-	require.Equal(t, got1.AppStoreApp.LastInstall.CommandUUID, cmdUUID)
+	require.Equal(t, got1.AppStoreApp.LastInstall.CommandUUID, installCmdUUID)
 	require.NotNil(t, got1.AppStoreApp.LastInstall.InstalledAt)
 
 	// Filter the self-service apps for that host
@@ -12530,14 +12585,28 @@ func (s *integrationMDMTestSuite) TestVPPApps() {
 		hostCount      int
 		deviceToken    string
 	}{
-		"iOS app install": {installHost: iOSHost, titleID: iOSTitleID, mdmClient: iOSMdmClient, app: iOSApp, hostCount: 1},
+		"iOS app install": {
+			installHost: iOSHost,
+			titleID:     iOSTitleID,
+			mdmClient:   iOSMdmClient,
+			app:         iOSApp,
+			hostCount:   1,
+		},
 		"iPadOS app install": {
-			installHost: iPadOSHost, titleID: iPadOSTitleID, mdmClient: iPadOSMdmClient, app: iPadOSApp,
-			extraAvailable: 1, hostCount: 1,
+			installHost:    iPadOSHost,
+			titleID:        iPadOSTitleID,
+			mdmClient:      iPadOSMdmClient,
+			app:            iPadOSApp,
+			extraAvailable: 1,
+			hostCount:      1,
 		},
 		"macOS app install": {
-			installHost: selfServiceHost, titleID: macOSTitleID, mdmClient: selfServiceDevice, app: macOSApp,
-			hostCount: 2, deviceToken: selfServiceToken,
+			installHost: selfServiceHost,
+			titleID:     macOSTitleID,
+			mdmClient:   selfServiceDevice,
+			app:         macOSApp,
+			hostCount:   2,
+			deviceToken: selfServiceToken,
 		},
 	}
 
@@ -12568,11 +12637,11 @@ func (s *integrationMDMTestSuite) TestVPPApps() {
 			require.NoError(t, err)
 			var fullCmd micromdm.CommandPayload
 			require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd))
-			cmdUUID = cmd.CommandUUID
+			installCmdUUID = cmd.CommandUUID
 
 			if install.deviceToken != "" {
 				var cmdResResp getMDMCommandResultsResponse
-				res := s.DoRawNoAuth("GET", fmt.Sprintf("/api/latest/fleet/device/%s/software/commands/%s/results", install.deviceToken, cmdUUID), nil, http.StatusOK)
+				res := s.DoRawNoAuth("GET", fmt.Sprintf("/api/latest/fleet/device/%s/software/commands/%s/results", install.deviceToken, installCmdUUID), nil, http.StatusOK)
 				err = json.NewDecoder(res.Body).Decode(&cmdResResp)
 				require.NoError(t, err)
 				require.Len(t, cmdResResp.Results, 0)
@@ -12590,9 +12659,9 @@ func (s *integrationMDMTestSuite) TestVPPApps() {
 				return res
 			}
 			require.Len(t, hostActivitiesResp.Activities, 1, "got activities: %v", activitiesToString(hostActivitiesResp.Activities))
-			assert.Equal(t, hostActivitiesResp.Activities[0].Type, fleet.ActivityInstalledAppStoreApp{}.ActivityName())
-			assert.EqualValues(t, 1, hostActivitiesResp.Count)
-			assert.JSONEq(
+			require.Equal(t, hostActivitiesResp.Activities[0].Type, fleet.ActivityInstalledAppStoreApp{}.ActivityName())
+			require.EqualValues(t, 1, hostActivitiesResp.Count)
+			require.JSONEq(
 				t,
 				fmt.Sprintf(
 					`{"host_id": %d, "host_display_name": "%s", "software_title": "%s", "app_store_id": "%s", "command_uuid": "%s", "status": "%s", "self_service": %v}`,
@@ -12600,7 +12669,7 @@ func (s *integrationMDMTestSuite) TestVPPApps() {
 					installHost.DisplayName(),
 					app.Name,
 					app.AdamID,
-					cmdUUID,
+					installCmdUUID,
 					fleet.SoftwareInstallPending,
 					install.deviceToken != "",
 				),
@@ -12610,29 +12679,44 @@ func (s *integrationMDMTestSuite) TestVPPApps() {
 			// Simulate successful installation on the host
 			cmd, err = mdmClient.Acknowledge(cmd.CommandUUID)
 			require.NoError(t, err)
-			// No further commands expected
-			assert.Nil(t, cmd)
+
+			// Process InstalledApplicationList command for install verification
+			s.runWorker()
+
+			cmd, err = mdmClient.Idle()
+			require.NoError(t, err)
+			for cmd != nil {
+				var fullCmd micromdm.CommandPayload
+				switch cmd.Command.RequestType { //nolint:gocritic // ignore singleCaseSwitch
+				case "InstalledApplicationList":
+					require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd))
+					cmd, err = mdmClient.AcknowledgeInstalledApplicationList(mdmClient.UUID, cmd.CommandUUID, []fleet.Software{{Name: app.Name, BundleIdentifier: app.BundleIdentifier, Version: app.LatestVersion}})
+					require.NoError(t, err)
+				default:
+					require.Fail(t, "unexpected command type", cmd.Command.RequestType)
+				}
+			}
 
 			if install.deviceToken != "" {
 				var cmdResResp getMDMCommandResultsResponse
-				res := s.DoRawNoAuth("GET", fmt.Sprintf("/api/latest/fleet/device/%s/software/commands/%s/results", install.deviceToken, cmdUUID), nil, http.StatusOK)
+				res := s.DoRawNoAuth("GET", fmt.Sprintf("/api/latest/fleet/device/%s/software/commands/%s/results", install.deviceToken, installCmdUUID), nil, http.StatusOK)
 				err = json.NewDecoder(res.Body).Decode(&cmdResResp)
 				require.NoError(t, err)
 				require.Len(t, cmdResResp.Results, 1)
 				require.Equal(t, "Acknowledged", cmdResResp.Results[0].Status)
 
 				s.DoRawNoAuth("GET", fmt.Sprintf("/api/latest/fleet/device/%s/software/commands/foobar/results", install.deviceToken), nil, http.StatusNotFound)
-				s.DoRawNoAuth("GET", fmt.Sprintf("/api/latest/fleet/device/foobar/software/commands/%s/results", cmdUUID), nil, http.StatusNotFound)
+				s.DoRawNoAuth("GET", fmt.Sprintf("/api/latest/fleet/device/foobar/software/commands/%s/results", installCmdUUID), nil, http.StatusNotFound)
 			}
 
 			listResp = listHostsResponse{}
 			s.DoJSON("GET", "/api/latest/fleet/hosts", nil, http.StatusOK, &listResp, "software_status", "installed", "team_id",
 				fmt.Sprint(team.ID), "software_title_id", fmt.Sprint(titleID))
-			assert.Len(t, listResp.Hosts, install.hostCount)
+			require.Len(t, listResp.Hosts, install.hostCount)
 			countResp = countHostsResponse{}
 			s.DoJSON("GET", "/api/latest/fleet/hosts/count", nil, http.StatusOK, &countResp, "software_status", "installed", "team_id",
 				fmt.Sprint(team.ID), "software_title_id", fmt.Sprint(titleID))
-			assert.Equal(t, install.hostCount, countResp.Count)
+			require.Equal(t, install.hostCount, countResp.Count)
 
 			s.lastActivityMatches(
 				fleet.ActivityInstalledAppStoreApp{}.ActivityName(),
@@ -12642,7 +12726,7 @@ func (s *integrationMDMTestSuite) TestVPPApps() {
 					installHost.DisplayName(),
 					app.Name,
 					app.AdamID,
-					cmdUUID,
+					installCmdUUID,
 					fleet.SoftwareInstalled,
 					install.deviceToken != "",
 				),
@@ -12664,7 +12748,7 @@ func (s *integrationMDMTestSuite) TestVPPApps() {
 					require.Empty(t, got1.AppStoreApp.Name) // Name is only present for installer packages
 					require.Equal(t, got1.AppStoreApp.Version, app.LatestVersion)
 					require.Equal(t, *got1.Status, fleet.SoftwareInstalled)
-					require.Equal(t, got1.AppStoreApp.LastInstall.CommandUUID, cmdUUID)
+					require.Equal(t, got1.AppStoreApp.LastInstall.CommandUUID, installCmdUUID)
 					require.NotNil(t, got1.AppStoreApp.LastInstall.InstalledAt)
 					foundInstalledApp = true
 				}
@@ -12722,7 +12806,11 @@ func (s *integrationMDMTestSuite) TestVPPAppPolicyAutomation() {
 	orbitHost := createOrbitEnrolledHost(t, "darwin", "nonmdm", s.ds)
 	mdmHost, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t)
 	setOrbitEnrollment(t, mdmHost, s.ds)
+	s.runWorker()
+	checkInstallFleetdCommandSent(t, mdmDevice, true)
 	mdmHost2, mdmDevice2 := createHostThenEnrollMDM(s.ds, s.server.URL, t)
+	s.runWorker()
+	checkInstallFleetdCommandSent(t, mdmDevice2, true)
 	key := setOrbitEnrollment(t, mdmHost2, s.ds)
 	mdmHost2.OrbitNodeKey = &key
 	selfServiceHost, selfServiceDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t)
@@ -13216,6 +13304,7 @@ func (s *integrationMDMTestSuite) TestVPPAppPolicyAutomation() {
 	require.Equal(t, uint(1), countPendingInstalls)
 
 	// send an idle request to grab the command uuid
+	s.runWorker()
 	var cmdUUID string
 	cmd, err := mdmDevice.Idle()
 	require.NoError(t, err)
@@ -13247,6 +13336,33 @@ func (s *integrationMDMTestSuite) TestVPPAppPolicyAutomation() {
 	_, err = mdmDevice.Acknowledge(cmd.CommandUUID)
 	require.NoError(t, err)
 
+	s.runWorker()
+
+	cmd, err = mdmDevice.Idle()
+	require.NoError(t, err)
+	for cmd != nil {
+		var fullCmd micromdm.CommandPayload
+		switch cmd.Command.RequestType { //nolint:gocritic // ignore singleCaseSwitch
+		case "InstalledApplicationList":
+			require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd))
+			cmd, err = mdmDevice.AcknowledgeInstalledApplicationList(mdmDevice.UUID, cmd.CommandUUID, []fleet.Software{{Name: macOSApp.Name, BundleIdentifier: macOSApp.BundleIdentifier, Version: macOSApp.LatestVersion}})
+			require.NoError(t, err)
+		default:
+			require.Fail(t, "unexpected command type", cmd.Command.RequestType)
+		}
+	}
+
+	s.runWorker()
+
+	// Shouldn't be any more pending installs
+	mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error {
+		return sqlx.GetContext(ctx, q, &countPendingInstalls, `SELECT COUNT(*)
+			FROM upcoming_activities
+			WHERE activity_type = 'vpp_app_install'
+			AND host_id = ?`, mdmHost.ID)
+	})
+	require.Zero(t, countPendingInstalls)
+
 	s.lastActivityMatchesExtended(
 		fleet.ActivityInstalledAppStoreApp{}.ActivityName(),
 		fmt.Sprintf(
@@ -13309,6 +13425,7 @@ func (s *integrationMDMTestSuite) TestVPPAppPolicyAutomation() {
 	)
 
 	// Process mdmHost2's vpp installation
+	s.runWorker()
 	cmd, err = mdmDevice2.Idle()
 	require.NoError(t, err)
 	require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd))
@@ -13335,9 +13452,33 @@ func (s *integrationMDMTestSuite) TestVPPAppPolicyAutomation() {
 		string(*hostActivitiesResp.Activities[0].Details),
 	)
 
-	_, err = mdmDevice.Acknowledge(cmd.CommandUUID)
+	_, err = mdmDevice2.Acknowledge(cmd.CommandUUID)
 	require.NoError(t, err)
 
+	s.runWorker()
+
+	cmd, err = mdmDevice2.Idle()
+	require.NoError(t, err)
+	for cmd != nil {
+		var fullCmd micromdm.CommandPayload
+		switch cmd.Command.RequestType { //nolint:gocritic // ignore singleCaseSwitch
+		case "InstalledApplicationList":
+			require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd))
+			cmd, err = mdmDevice2.AcknowledgeInstalledApplicationList(mdmDevice2.UUID, cmd.CommandUUID, []fleet.Software{{Name: macOSApp.Name, BundleIdentifier: macOSApp.BundleIdentifier, Version: macOSApp.LatestVersion}})
+			require.NoError(t, err)
+		default:
+			require.Fail(t, "unexpected command type", cmd.Command.RequestType)
+		}
+	}
+
+	mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error {
+		return sqlx.GetContext(ctx, q, &countPendingInstalls, `SELECT COUNT(*)
+			FROM upcoming_activities
+			WHERE activity_type = 'vpp_app_install'
+			AND host_id = ?`, mdmHost2.ID)
+	})
+	require.Zero(t, countPendingInstalls)
+
 	s.lastActivityMatchesExtended(
 		fleet.ActivityInstalledAppStoreApp{}.ActivityName(),
 		fmt.Sprintf(
@@ -16351,6 +16492,9 @@ func (s *integrationMDMTestSuite) TestCancelUpcomingActivity() {
 	key := setOrbitEnrollment(t, mdmHost, s.ds)
 	mdmHost.OrbitNodeKey = &key
 
+	s.runWorker()
+	checkInstallFleetdCommandSent(t, mdmDevice, true)
+
 	// Add serial number to our fake Apple server
 	s.appleVPPConfigSrvConfig.SerialNumbers = append(s.appleVPPConfigSrvConfig.SerialNumbers, mdmHost.HardwareSerial)
 
@@ -16419,6 +16563,22 @@ func (s *integrationMDMTestSuite) TestCancelUpcomingActivity() {
 		}
 	}
 
+	s.runWorker()
+
+	cmd, err = mdmDevice.Idle()
+	require.NoError(t, err)
+	for cmd != nil {
+		var fullCmd micromdm.CommandPayload
+		switch cmd.Command.RequestType { //nolint:gocritic // ignore singleCaseSwitch
+		case "InstalledApplicationList":
+			require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd))
+			cmd, err = mdmDevice.AcknowledgeInstalledApplicationList(mdmDevice.UUID, cmd.CommandUUID, []fleet.Software{{Name: addedApp.Name, BundleIdentifier: addedApp.BundleIdentifier, Version: addedApp.LatestVersion}})
+			require.NoError(t, err)
+		default:
+			require.Fail(t, "unexpected command type", cmd.Command.RequestType)
+		}
+	}
+
 	// record a failure for software install
 	s.Do("POST", "/api/fleet/orbit/software_install/result", json.RawMessage(fmt.Sprintf(`{
 			"orbit_node_key": %q, "install_uuid": %q, "pre_install_condition_output": "ok", "install_script_exit_code": 1, "install_script_output": "fail"
@@ -16504,6 +16664,10 @@ func (s *integrationMDMTestSuite) TestCancelUpcomingActivity() {
 			require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd))
 			cmd, err = mdmDevice.Acknowledge(cmd.CommandUUID)
 			require.NoError(t, err)
+		case "InstalledApplicationList":
+			require.NoError(t, plist.Unmarshal(cmd.Raw, &fullCmd))
+			cmd, err = mdmDevice.AcknowledgeInstalledApplicationList(mdmDevice.UUID, cmd.CommandUUID, []fleet.Software{{Name: addedApp.Name, BundleIdentifier: addedApp.BundleIdentifier, Version: addedApp.LatestVersion}})
+			require.NoError(t, err)
 		default:
 			require.Fail(t, "unexpected command", cmd.Command.RequestType)
 		}

server/service/testing_utils.go

@@ -406,13 +406,15 @@ func RunServerForTestsWithServiceWithDS(t *testing.T, ctx context.Context, ds fl
 		scepStorage := opts[0].SCEPStorage
 		commander := apple_mdm.NewMDMAppleCommander(mdmStorage, mdmPusher)
 		if mdmStorage != nil && scepStorage != nil {
+			checkInAndCommand := NewMDMAppleCheckinAndCommandService(ds, commander, logger)
+			checkInAndCommand.RegisterResultsHandler("InstalledApplicationList", NewInstalledApplicationListResultsHandler(ds, commander, logger, cfg.Server.VPPVerifyTimeout, cfg.Server.VPPVerifyRequestDelay))
 			err := RegisterAppleMDMProtocolServices(
 				rootMux,
 				cfg.MDM,
 				mdmStorage,
 				scepStorage,
 				logger,
-				NewMDMAppleCheckinAndCommandService(ds, commander, logger),
+				checkInAndCommand,
 				&MDMAppleDDMService{
 					ds:     ds,
 					logger: logger,

server/worker/vpp_verification.go

@@ -0,0 +1,90 @@
+package worker
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	apple_mdm "github.com/fleetdm/fleet/v4/server/mdm/apple"
+	kitlog "github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+	"github.com/google/uuid"
+)
+
+const VPPVerificationJobName = "vpp_verification"
+
+type VPPVerificationTask string
+
+const VerifyVPPTask VPPVerificationTask = "verify_vpp_installs"
+
+type VPPVerification struct {
+	Datastore fleet.Datastore
+	Commander *apple_mdm.MDMAppleCommander
+	Log       kitlog.Logger
+}
+
+func (v *VPPVerification) Name() string {
+	return VPPVerificationJobName
+}
+
+type vppVerificationArgs struct {
+	Task                    VPPVerificationTask `json:"task"`
+	HostUUID                string              `json:"host_uuid"`
+	VerificationCommandUUID string              `json:"verification_command_uuid"`
+}
+
+func (v *VPPVerification) Run(ctx context.Context, argsJSON json.RawMessage) error {
+	var args vppVerificationArgs
+	if err := json.Unmarshal(argsJSON, &args); err != nil {
+		return ctxerr.Wrap(ctx, err, "unmarshal args")
+	}
+
+	switch args.Task {
+	case VerifyVPPTask:
+		err := v.verifyVPPInstalls(ctx, args.HostUUID, args.VerificationCommandUUID)
+		return ctxerr.Wrap(ctx, err, "running migrate VPP token task")
+
+	default:
+		return ctxerr.Errorf(ctx, "unknown task: %v", args.Task)
+	}
+}
+
+func (v *VPPVerification) verifyVPPInstalls(ctx context.Context, hostUUID, verificationCommandUUID string) error {
+	pendingCmds, err := v.Datastore.GetAcknowledgedMDMCommandsByHost(ctx, hostUUID, "InstalledApplicationList")
+	if err != nil {
+		return ctxerr.Wrap(ctx, err, "get pending mdm commands by host")
+	}
+	// Only send a new list command if none are in flight. If there's one in
+	// flight, the install will be verified by that one.
+	if len(pendingCmds) == 0 {
+		newListCmdUUID := fleet.RefetchVPPAppInstallsCommandUUIDPrefix + uuid.NewString()
+		if err := v.Datastore.UpdateVPPInstallVerificationCommandByVerifyUUID(ctx, verificationCommandUUID, newListCmdUUID); err != nil {
+			return ctxerr.Wrap(ctx, err, "update install record")
+		}
+		err := v.Commander.InstalledApplicationList(ctx, []string{hostUUID}, newListCmdUUID, true)
+		if err != nil {
+			return ctxerr.Wrap(ctx, err, "sending installed application list command in verify")
+		}
+		level.Debug(v.Log).Log("msg", "new installed application list command sent", "uuid", newListCmdUUID)
+	}
+
+	return nil
+}
+
+func QueueVPPInstallVerificationJob(ctx context.Context, ds fleet.Datastore, logger kitlog.Logger, task VPPVerificationTask, requestDelay time.Duration, hostUUID, verificationCommandUUID string) error {
+	args := &vppVerificationArgs{
+		Task:                    task,
+		HostUUID:                hostUUID,
+		VerificationCommandUUID: verificationCommandUUID,
+	}
+
+	job, err := QueueJobWithDelay(ctx, ds, VPPVerificationJobName, args, requestDelay)
+	if err != nil {
+		return ctxerr.Wrap(ctx, err, "queueing job")
+	}
+
+	level.Debug(logger).Log("job_id", job.ID)
+	return nil
+}