From 95df7e2b0ba3b61f157d8bb00c54eeb00b8978da Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Mon, 25 Mar 2024 17:32:27 -0300 Subject: [PATCH] implement DDM cron and protocol bits (#17791) for #17399 --- cmd/fleet/cron.go | 3 + server/datastore/mysql/apple_mdm.go | 373 +++++++++++------- server/datastore/mysql/apple_mdm_test.go | 2 - server/datastore/mysql/mdm.go | 3 +- .../tables/20240314150853_AddDDMTables.go | 25 +- server/datastore/mysql/schema.sql | 13 +- server/fleet/apple_mdm.go | 34 +- server/fleet/datastore.go | 11 +- server/mock/datastore_mock.go | 30 +- server/service/apple_mdm.go | 112 +++++- server/service/integration_mdm_test.go | 291 ++++++++++++-- 11 files changed, 628 insertions(+), 269 deletions(-) diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index ad20a3278d..42fca14d17 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -1033,6 +1033,9 @@ func newMDMProfileManager( schedule.WithJob("manage_apple_profiles", func(ctx context.Context) error { return service.ReconcileAppleProfiles(ctx, ds, commander, logger) }), + schedule.WithJob("manage_apple_declarations", func(ctx context.Context) error { + return service.ReconcileAppleDeclarations(ctx, ds, commander, logger) + }), schedule.WithJob("manage_windows_profiles", func(ctx context.Context) error { return service.ReconcileWindowsProfiles(ctx, ds, logger) }), diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index ac0801aad8..10d1b27faf 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -16,8 +16,8 @@ import ( "github.com/fleetdm/fleet/v4/server/mdm/nanodep/godep" "github.com/fleetdm/fleet/v4/server/mdm/nanomdm/mdm" "github.com/fleetdm/fleet/v4/server/ptr" - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/google/uuid" "github.com/jmoiron/sqlx" ) @@ -240,7 +240,7 @@ SELECT FROM mdm_apple_declarations WHERE - declaration_uuid = ? AND category = 'com.apple.configuration'` + declaration_uuid = ?` var res fleet.MDMAppleDeclaration err := sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, declUUID) @@ -1490,6 +1490,8 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB( return nil } + appleMDMProfilesDesiredStateQuery := generateDesiredStateQuery("profile") + // TODO(mna): the conditions here (and in toRemoveStmt) are subtly different // than the ones in ListMDMAppleProfilesToInstall/Remove, so I'm keeping // those statements distinct to avoid introducing a subtle bug, but we should @@ -1720,20 +1722,30 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB( return nil } -const appleMDMProfilesDesiredStateQuery = ` - -- non label-based profiles +// mdmEntityTypeToTable tracks what table should be used in the templates for +// SQL statements based on the given entity type. +var mdmEntityTypeToTable = map[string]string{ + "declaration": "declaration", + "profile": "configuration_profile", +} + +// generateDesiredStateQuery generates a query string that represents the +// desired state of an Apple entity based on its type (profile or declaration) +func generateDesiredStateQuery(entityType string) string { + return fmt.Sprintf(` + -- non label-based entities SELECT - macp.profile_uuid, + mae.%[1]s_uuid, h.uuid as host_uuid, - macp.identifier as profile_identifier, - macp.name as profile_name, - macp.checksum as checksum, - 0 as count_profile_labels, + mae.identifier as %[1]s_identifier, + mae.name as %[1]s_name, + mae.checksum as checksum, + 0 as count_%[1]s_labels, 0 as count_host_labels FROM - mdm_apple_configuration_profiles macp + mdm_apple_%[2]ss mae JOIN hosts h - ON h.team_id = macp.team_id OR (h.team_id IS NULL AND macp.team_id = 0) + ON h.team_id = mae.team_id OR (h.team_id IS NULL AND mae.team_id = 0) JOIN nano_enrollments ne ON ne.device_id = h.uuid WHERE @@ -1742,81 +1754,146 @@ const appleMDMProfilesDesiredStateQuery = ` ne.type = 'Device' AND NOT EXISTS ( SELECT 1 - FROM mdm_configuration_profile_labels mcpl - WHERE mcpl.apple_profile_uuid = macp.profile_uuid + FROM mdm_%[2]s_labels mel + WHERE mel.apple_%[1]s_uuid = mae.%[1]s_uuid ) AND - ( %s ) + ( %[3]s ) UNION - -- label-based profiles where the host is a member of all the labels + -- label-based entities where the host is a member of all the labels SELECT - macp.profile_uuid, + mae.%[1]s_uuid, h.uuid as host_uuid, - macp.identifier as profile_identifier, - macp.name as profile_name, - macp.checksum as checksum, - COUNT(*) as count_profile_labels, + mae.identifier as %[1]s_identifier, + mae.name as %[1]s_name, + mae.checksum as checksum, + COUNT(*) as count_%[1]s_labels, COUNT(lm.label_id) as count_host_labels FROM - mdm_apple_configuration_profiles macp + mdm_apple_%[2]ss mae JOIN hosts h - ON h.team_id = macp.team_id OR (h.team_id IS NULL AND macp.team_id = 0) + ON h.team_id = mae.team_id OR (h.team_id IS NULL AND mae.team_id = 0) JOIN nano_enrollments ne ON ne.device_id = h.uuid - JOIN mdm_configuration_profile_labels mcpl - ON mcpl.apple_profile_uuid = macp.profile_uuid + JOIN mdm_%[2]s_labels mel + ON mel.apple_%[1]s_uuid = mae.%[1]s_uuid LEFT OUTER JOIN label_membership lm - ON lm.label_id = mcpl.label_id AND lm.host_id = h.id + ON lm.label_id = mel.label_id AND lm.host_id = h.id WHERE h.platform = 'darwin' AND ne.enabled = 1 AND ne.type = 'Device' AND - ( %s ) + ( %[3]s ) GROUP BY - macp.profile_uuid, h.uuid, macp.identifier, macp.name, macp.checksum + mae.%[1]s_uuid, h.uuid, mae.identifier, mae.name, mae.checksum HAVING - count_profile_labels > 0 AND count_host_labels = count_profile_labels -` + count_%[1]s_labels > 0 AND count_host_labels = count_%[1]s_labels + + `, entityType, mdmEntityTypeToTable[entityType], "%s") +} + +// generateEntitiesToInstallQuery is a set difference between: +// +// - Set A (ds), the "desired state", can be obtained from a JOIN between +// mdm_apple_x and hosts. +// +// - Set B, the "current state" given by host_mdm_apple_x. +// +// A - B gives us the entities that need to be installed: +// +// - entities that are in A but not in B +// +// - entities which contents have changed, but their identifier are +// the same (by checking the checksums) +// +// - entities that are in A and in B, but with an operation type of +// "remove", regardless of the status. (technically, if status is NULL then +// the entity should be already installed - it has not been queued for +// remove yet -, and same if status is failed, but the proper thing to do +// with it would be to remove the row, not return it as "to install". For +// simplicity of implementation here (and to err on the safer side - the +// entity's content could've changed), we'll return it as "to install" for +// now, which will cause the row to be updated with the correct operation +// type and status). +// +// - entities that are in A and in B, with an operation type of "install" +// and a NULL status. Other statuses mean that the operation is already in +// flight (pending), the operation has been completed but is still subject +// to independent verification by Fleet (verifying), or has reached a terminal +// state (failed or verified). If the entity's content is edited, all +// relevant hosts will be marked as status NULL so that it gets +// re-installed. +// +// Note that for label-based entities, only fully-satisfied entities are +// considered for installation. This means that a broken label-based entity, +// where one of the labels does not exist anymore, will not be considered for +// installation. +func generateEntitiesToInstallQuery(entityType string) string { + return fmt.Sprintf(` + ( %[3]s ) as ds + LEFT JOIN host_mdm_apple_%[1]ss hmae + ON hmae.%[1]s_uuid = ds.%[1]s_uuid AND hmae.host_uuid = ds.host_uuid + WHERE + -- entity has been updated + ( hmae.checksum != ds.checksum ) OR + -- entity in A but not in B + ( hmae.%[1]s_uuid IS NULL AND hmae.host_uuid IS NULL ) OR + -- entities in A and B but with operation type "remove" + ( hmae.host_uuid IS NOT NULL AND ( hmae.operation_type = ? OR hmae.operation_type IS NULL ) ) OR + -- entities in A and B with operation type "install" and NULL status + ( hmae.host_uuid IS NOT NULL AND hmae.operation_type = ? AND hmae.status IS NULL ) +`, entityType, mdmEntityTypeToTable[entityType], fmt.Sprintf(generateDesiredStateQuery(entityType), "TRUE", "TRUE")) +} + +// generateEntitiesToRemoveQuery is a set difference between: +// +// - Set A (ds), the "desired state", can be obtained from a JOIN between +// mdm_apple_configuration_x and hosts. +// +// - Set B, the "current state" given by host_mdm_apple_x. +// +// B - A gives us the entities that need to be removed: +// +// - entities that are in B but not in A, except those with operation type +// "remove" and a terminal state (failed) or a state indicating +// that the operation is in flight (pending) or the operation has been completed +// but is still subject to independent verification by Fleet (verifying) +// or the operation has been completed and independenly verified by Fleet (verified). +// +// Any other case are entities that are in both B and A, and as such are +// processed by the generateEntitiesToInstallQuery query (since they are in +// both, their desired state is necessarily to be installed). +// +// Note that for label-based entities, only those that are fully-sastisfied +// by the host are considered for install (are part of the desired state used +// to compute the ones to remove). However, as a special case, a broken +// label-based entity will NOT be removed from a host where it was +// previously installed. However, if a host used to satisfy a label-based +// entity but no longer does (and that label-based entity is not "broken"), +// the entity will be removed from the host. +func generateEntitiesToRemoveQuery(entityType string) string { + return fmt.Sprintf(` + ( %[3]s ) as ds + RIGHT JOIN host_mdm_apple_%[1]ss hmae + ON hmae.%[1]s_uuid = ds.%[1]s_uuid AND hmae.host_uuid = ds.host_uuid + WHERE + -- entities that are in B but not in A + ds.%[1]s_uuid IS NULL AND ds.host_uuid IS NULL AND + -- except "remove" operations in a terminal state or already pending + ( hmae.operation_type IS NULL OR hmae.operation_type != ? OR hmae.status IS NULL ) AND + -- except "would be removed" entities if they are a broken label-based entities + NOT EXISTS ( + SELECT 1 + FROM mdm_%[2]s_labels mcpl + WHERE + mcpl.apple_%[1]s_uuid = hmae.%[1]s_uuid AND + mcpl.label_id IS NULL + ) +`, entityType, mdmEntityTypeToTable[entityType], fmt.Sprintf(generateDesiredStateQuery(entityType), "TRUE", "TRUE")) +} func (ds *Datastore) ListMDMAppleProfilesToInstall(ctx context.Context) ([]*fleet.MDMAppleProfilePayload, error) { - // The query below is a set difference between: - // - // - Set A (ds), the "desired state", can be obtained from a JOIN between - // mdm_apple_configuration_profiles and hosts. - // - // - Set B, the "current state" given by host_mdm_apple_profiles. - // - // A - B gives us the profiles that need to be installed: - // - // - profiles that are in A but not in B - // - // - profiles which contents have changed, but their identifier are - // the same (by checking the checksums) - // - // - profiles that are in A and in B, but with an operation type of - // "remove", regardless of the status. (technically, if status is NULL then - // the profile should be already installed - it has not been queued for - // remove yet -, and same if status is failed, but the proper thing to do - // with it would be to remove the row, not return it as "to install". For - // simplicity of implementation here (and to err on the safer side - the - // profile's content could've changed), we'll return it as "to install" for - // now, which will cause the row to be updated with the correct operation - // type and status). - // - // - profiles that are in A and in B, with an operation type of "install" - // and a NULL status. Other statuses mean that the operation is already in - // flight (pending), the operation has been completed but is still subject - // to independent verification by Fleet (verifying), or has reached a terminal - // state (failed or verified). If the profile's content is edited, all - // relevant hosts will be marked as status NULL so that it gets - // re-installed. - // - // Note that for label-based profiles, only fully-satisfied profiles are - // considered for installation. This means that a broken label-based profile, - // where one of the labels does not exist anymore, will not be considered for - // installation. - query := fmt.Sprintf(` SELECT ds.profile_uuid, @@ -1824,82 +1901,26 @@ func (ds *Datastore) ListMDMAppleProfilesToInstall(ctx context.Context) ([]*flee ds.profile_identifier, ds.profile_name, ds.checksum - FROM ( %s ) as ds - LEFT JOIN host_mdm_apple_profiles hmap - ON hmap.profile_uuid = ds.profile_uuid AND hmap.host_uuid = ds.host_uuid - WHERE - -- profile has been updated - ( hmap.checksum != ds.checksum ) OR - -- profiles in A but not in B - ( hmap.profile_uuid IS NULL AND hmap.host_uuid IS NULL ) OR - -- profiles in A and B but with operation type "remove" - ( hmap.host_uuid IS NOT NULL AND ( hmap.operation_type = ? OR hmap.operation_type IS NULL ) ) OR - -- profiles in A and B with operation type "install" and NULL status - ( hmap.host_uuid IS NOT NULL AND hmap.operation_type = ? AND hmap.status IS NULL ) -`, fmt.Sprintf(appleMDMProfilesDesiredStateQuery, "TRUE", "TRUE")) - + FROM %s `, + generateEntitiesToInstallQuery("profile")) var profiles []*fleet.MDMAppleProfilePayload err := sqlx.SelectContext(ctx, ds.reader(ctx), &profiles, query, fleet.MDMOperationTypeRemove, fleet.MDMOperationTypeInstall) return profiles, err } func (ds *Datastore) ListMDMAppleProfilesToRemove(ctx context.Context) ([]*fleet.MDMAppleProfilePayload, error) { - // The query below is a set difference between: - // - // - Set A (ds), the "desired state", can be obtained from a JOIN between - // mdm_apple_configuration_profiles and hosts. - // - // - Set B, the "current state" given by host_mdm_apple_profiles. - // - // B - A gives us the profiles that need to be removed: - // - // - profiles that are in B but not in A, except those with operation type - // "remove" and a terminal state (failed) or a state indicating - // that the operation is in flight (pending) or the operation has been completed - // but is still subject to independent verification by Fleet (verifying) - // or the operation has been completed and independenly verified by Fleet (verified). - // - // Any other case are profiles that are in both B and A, and as such are - // processed by the ListMDMAppleProfilesToInstall method (since they are in - // both, their desired state is necessarily to be installed). - // - // Note that for label-based profiles, only those that are fully-sastisfied - // by the host are considered for install (are part of the desired state used - // to compute the ones to remove). However, as a special case, a broken - // label-based profile will NOT be removed from a host where it was - // previously installed. However, if a host used to satisfy a label-based - // profile but no longer does (and that label-based profile is not "broken"), - // the profile will be removed from the host. - query := fmt.Sprintf(` SELECT - hmap.profile_uuid, - hmap.profile_identifier, - hmap.profile_name, - hmap.host_uuid, - hmap.checksum, - hmap.operation_type, - COALESCE(hmap.detail, '') as detail, - hmap.status, - hmap.command_uuid - FROM ( %s ) as ds - RIGHT JOIN host_mdm_apple_profiles hmap - ON hmap.profile_uuid = ds.profile_uuid AND hmap.host_uuid = ds.host_uuid - WHERE - -- profiles that are in B but not in A - ds.profile_uuid IS NULL AND ds.host_uuid IS NULL AND - -- except "remove" operations in a terminal state or already pending - ( hmap.operation_type IS NULL OR hmap.operation_type != ? OR hmap.status IS NULL ) AND - -- except "would be removed" profiles if they are a broken label-based profile - NOT EXISTS ( - SELECT 1 - FROM mdm_configuration_profile_labels mcpl - WHERE - mcpl.apple_profile_uuid = hmap.profile_uuid AND - mcpl.label_id IS NULL - ) -`, fmt.Sprintf(appleMDMProfilesDesiredStateQuery, "TRUE", "TRUE")) - + hmae.profile_uuid, + hmae.profile_identifier, + hmae.profile_name, + hmae.host_uuid, + hmae.checksum, + hmae.operation_type, + COALESCE(hmae.detail, '') as detail, + hmae.status, + hmae.command_uuid + FROM %s`, generateEntitiesToRemoveQuery("profile")) var profiles []*fleet.MDMAppleProfilePayload err := sqlx.SelectContext(ctx, ds.reader(ctx), &profiles, query, fleet.MDMOperationTypeRemove) return profiles, err @@ -3166,14 +3187,13 @@ INSERT INTO mdm_apple_declarations ( declaration_uuid, identifier, name, - category, raw_json, checksum, uploaded_at, team_id ) VALUES ( - ?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP(),? + ?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP(),? ) ON DUPLICATE KEY UPDATE uploaded_at = IF(checksum = VALUES(checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP()), @@ -3283,7 +3303,6 @@ WHERE declUUID, d.Identifier, d.Name, - d.Category, d.RawJSON, checksum, declTeamID); err != nil || strings.HasPrefix(ds.testBatchSetMDMAppleProfilesErr, "insert") { @@ -3320,11 +3339,10 @@ INSERT INTO mdm_apple_declarations ( team_id, identifier, name, - category, raw_json, checksum, uploaded_at) -(SELECT ?,?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP() FROM DUAL WHERE +(SELECT ?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP() FROM DUAL WHERE NOT EXISTS ( SELECT 1 FROM mdm_windows_configuration_profiles WHERE name = ? AND team_id = ? ) AND NOT EXISTS ( @@ -3339,7 +3357,7 @@ INSERT INTO mdm_apple_declarations ( err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { res, err := tx.ExecContext(ctx, stmt, - declUUID, tmID, declaration.Identifier, declaration.Name, declaration.Category, declaration.RawJSON, checksum, declaration.Name, tmID, declaration.Name, tmID) + declUUID, tmID, declaration.Identifier, declaration.Name, declaration.RawJSON, checksum, declaration.Name, tmID, declaration.Name, tmID) if err != nil { switch { case isDuplicate(err): @@ -3473,8 +3491,7 @@ func (ds *Datastore) MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID s const stmt = ` SELECT HEX(mad.checksum) as checksum, - mad.identifier, - mad.category + mad.identifier FROM host_mdm_apple_declarations hmad JOIN mdm_apple_declarations mad ON mad.declaration_uuid = hmad.declaration_uuid @@ -3489,7 +3506,7 @@ WHERE return res, nil } -func (ds *Datastore) MDMAppleDDMDeclarationsResponse(ctx context.Context, declarationType fleet.MDMAppleDeclarationCategory, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) { +func (ds *Datastore) MDMAppleDDMDeclarationsResponse(ctx context.Context, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) { // TODO: When hosts table is indexed by uuid, consider joining on hosts to ensure that the // declaration for the host's current team is returned. In the case where the specified // identifier is not unique to the team, the cron should ensure that any conflicting @@ -3501,15 +3518,77 @@ FROM host_mdm_apple_declarations hmad JOIN mdm_apple_declarations mad ON hmad.declaration_uuid = mad.declaration_uuid WHERE - host_uuid = ? AND identifier = ? AND category = ? AND operation_type = ?` + host_uuid = ? AND identifier = ? AND operation_type = ?` var res fleet.MDMAppleDeclaration - if err := sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, hostUUID, identifier, declarationType, fleet.MDMOperationTypeInstall); err != nil { + if err := sqlx.GetContext(ctx, ds.reader(ctx), &res, stmt, hostUUID, identifier, fleet.MDMOperationTypeInstall); err != nil { if err == sql.ErrNoRows { - return nil, notFound(string(declarationType)).WithName(identifier) + return nil, notFound("MDMAppleDeclaration").WithName(identifier) } return nil, ctxerr.Wrap(ctx, err, "get ddm declarations response") } return &res, nil } + +func (ds *Datastore) MDMAppleBatchInsertHostDeclarations(ctx context.Context, changedDeclarations []*fleet.MDMAppleHostDeclaration) error { + baseStmt := ` + INSERT INTO host_mdm_apple_declarations + (host_uuid, status, operation_type, checksum, declaration_uuid, declaration_identifier, declaration_name) + VALUES + %s + ON DUPLICATE KEY UPDATE + status = VALUES(status), + operation_type = VALUES(operation_type), + checksum = VALUES(checksum) + ` + var placeholders strings.Builder + var args []any + for _, d := range changedDeclarations { + placeholders.WriteString("(?, 'pending', ?, ?, ?, ?, ?),") + args = append(args, d.HostUUID, d.OperationType, d.Checksum, d.DeclarationUUID, d.Identifier, d.Name) + } + _, err := ds.writer(ctx).ExecContext( + ctx, + fmt.Sprintf(baseStmt, strings.TrimSuffix(placeholders.String(), ",")), + args..., + ) + return ctxerr.Wrap(ctx, err, "inserting changed host declaration state") +} + +func (ds *Datastore) MDMAppleGetHostsWithChangedDeclarations(ctx context.Context) ([]*fleet.MDMAppleHostDeclaration, error) { + stmt := fmt.Sprintf(` + ( + SELECT + ds.host_uuid, + 'install' as operation_type, + ds.checksum, + ds.declaration_uuid, + ds.declaration_identifier as identifier, + ds.declaration_name as name + FROM + %s + ) + UNION ALL + ( + SELECT + hmae.host_uuid, + 'remove' as operation_type, + hmae.checksum, + hmae.declaration_uuid, + hmae.declaration_identifier as identifier, + hmae.declaration_name as name + FROM + %s + ) + `, + generateEntitiesToInstallQuery("declaration"), + generateEntitiesToRemoveQuery("declaration"), + ) + + var decls []*fleet.MDMAppleHostDeclaration + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &decls, stmt, fleet.MDMOperationTypeRemove, fleet.MDMOperationTypeInstall, fleet.MDMOperationTypeRemove); err != nil { + return nil, ctxerr.Wrap(ctx, err, "running sql statement") + } + return decls, nil +} diff --git a/server/datastore/mysql/apple_mdm_test.go b/server/datastore/mysql/apple_mdm_test.go index 1bfe5f6574..6eed346a69 100644 --- a/server/datastore/mysql/apple_mdm_test.go +++ b/server/datastore/mysql/apple_mdm_test.go @@ -1062,7 +1062,6 @@ func expectAppleDeclarations( require.Equal(t, wantD.Name, gotD.Name) require.Equal(t, wantD.Identifier, gotD.Identifier) require.Equal(t, wantD.Labels, gotD.Labels) - require.Equal(t, wantD.Category, gotD.Category) } return m } @@ -1258,7 +1257,6 @@ func declForTest(name, identifier, payloadContent string, labels ...*fleet.Label decl := &fleet.MDMAppleDeclaration{ RawJSON: declBytes, - Category: fleet.MDMAppleDeclarativeConfiguration, Identifier: fmt.Sprintf("com.fleet.config%s", identifier), Name: name, } diff --git a/server/datastore/mysql/mdm.go b/server/datastore/mysql/mdm.go index 21ff675ed1..6316bddd31 100644 --- a/server/datastore/mysql/mdm.go +++ b/server/datastore/mysql/mdm.go @@ -182,7 +182,6 @@ FROM ( uploaded_at FROM mdm_apple_declarations WHERE team_id = ? - AND category <> ? ) as combined_profiles ` @@ -202,7 +201,7 @@ FROM ( fleetNames = append(fleetNames, k) } - args := []any{globalOrTeamID, fleetIdentifiers, globalOrTeamID, fleetNames, globalOrTeamID, fleet.MDMAppleDeclarativeActivation} + args := []any{globalOrTeamID, fleetIdentifiers, globalOrTeamID, fleetNames, globalOrTeamID} stmt, args := appendListOptionsWithCursorToSQL(selectStmt, args, &opt) stmt, args, err := sqlx.In(stmt, args...) diff --git a/server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go b/server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go index b220e91b33..29f947a9fa 100644 --- a/server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go +++ b/server/datastore/mysql/migrations/tables/20240314150853_AddDDMTables.go @@ -11,24 +11,6 @@ func init() { func Up_20240314150853(tx *sql.Tx) error { _, err := tx.Exec(` -CREATE TABLE mdm_apple_declaration_categories ( - declaration_category varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, - PRIMARY KEY (declaration_category) -) - `) - if err != nil { - return fmt.Errorf("creating mdm_apple_declaration_categories table: %w", err) - } - - _, err = tx.Exec(` - INSERT INTO mdm_apple_declaration_categories - VALUES ('com.apple.configuration'), ('com.apple.activation') - `) - if err != nil { - return fmt.Errorf("inserting default values into mdm_apple_declaration_categories table: %w", err) - } - - _, err = tx.Exec(` CREATE TABLE mdm_apple_declarations ( -- declaration_uuid is used as the primary key of the declaration declaration_uuid varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', @@ -42,10 +24,6 @@ CREATE TABLE mdm_apple_declarations ( -- name is the name of the declaration name varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, - -- category is the category of the declaration (activation, - -- declaration, management or asset) - category varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, - -- raw_json contains a JSON blob with the declaration contents raw_json json NOT NULL, @@ -57,8 +35,7 @@ CREATE TABLE mdm_apple_declarations ( PRIMARY KEY (declaration_uuid), UNIQUE KEY idx_mdm_apple_declaration_team_identifier (team_id, identifier), - UNIQUE KEY idx_mdm_apple_declaration_team_name (team_id, name), - CONSTRAINT mdm_apple_declaration_category FOREIGN KEY (category) REFERENCES mdm_apple_declaration_categories (declaration_category) ON DELETE CASCADE + UNIQUE KEY idx_mdm_apple_declaration_team_name (team_id, name) ) `) if err != nil { diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index 3882a3061e..99b8644db5 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -660,29 +660,18 @@ CREATE TABLE `mdm_apple_declaration_activation_references` ( /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; -CREATE TABLE `mdm_apple_declaration_categories` ( - `declaration_category` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, - PRIMARY KEY (`declaration_category`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -/*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `mdm_apple_declaration_categories` VALUES ('com.apple.activation'),('com.apple.configuration'); -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; CREATE TABLE `mdm_apple_declarations` ( `declaration_uuid` varchar(37) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', `team_id` int(10) unsigned NOT NULL DEFAULT '0', `identifier` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, `name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, - `category` varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL, `raw_json` json NOT NULL, `checksum` binary(16) NOT NULL, `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `uploaded_at` timestamp NULL DEFAULT NULL, PRIMARY KEY (`declaration_uuid`), UNIQUE KEY `idx_mdm_apple_declaration_team_identifier` (`team_id`,`identifier`), - UNIQUE KEY `idx_mdm_apple_declaration_team_name` (`team_id`,`name`), - KEY `mdm_apple_declaration_category` (`category`), - CONSTRAINT `mdm_apple_declaration_category` FOREIGN KEY (`category`) REFERENCES `mdm_apple_declaration_categories` (`declaration_category`) ON DELETE CASCADE + UNIQUE KEY `idx_mdm_apple_declaration_team_name` (`team_id`,`name`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; diff --git a/server/fleet/apple_mdm.go b/server/fleet/apple_mdm.go index 78b6cda55d..093656e58a 100644 --- a/server/fleet/apple_mdm.go +++ b/server/fleet/apple_mdm.go @@ -450,6 +450,8 @@ const ( DEPAssignProfileResponseFailed DEPAssignProfileResponseStatus = "FAILED" ) +const MDMAppleDeclarationUUIDPrefix = "d" + // NanoEnrollment represents a row in the nano_enrollments table managed by // nanomdm. It is meant to be used internally by the server, not to be returned // as part of endpoints, and as a precaution its json-encoding is explicitly @@ -533,25 +535,6 @@ type SCEPIdentityAssociation struct { RenewCommandUUID string `db:"renew_command_uuid"` } -// MDMAppleDeclarationCategory is the type for the supported declaration types. -type MDMAppleDeclarationCategory string - -const ( - // MDMAppleConfigurationDeclaration is the value for [configuration][1] declarations - // - // [1]: https://developer.apple.com/documentation/devicemanagement/declarations#3813088 - MDMAppleDeclarativeConfiguration MDMAppleDeclarationCategory = "com.apple.configuration" - - // MDMAppleActivationConfiguration is the value for [activation][1] declarations - // - // [1]: https://developer.apple.com/documentation/devicemanagement/declarations#3829708 - MDMAppleDeclarativeActivation MDMAppleDeclarationCategory = "com.apple.activation" - - // MDMAppleDeclarationUUIDPrefix is the prefix used to differentiate declaration uuids - // from legacy Apple profile uuids and Windows profile uuids. - MDMAppleDeclarationUUIDPrefix = "d" -) - // MDMAppleDeclaration represents a DDM JSON declaration. type MDMAppleDeclaration struct { // DeclarationUUID is the unique identifier of the declaration in @@ -572,10 +555,6 @@ type MDMAppleDeclaration struct { // Fleet requires that Name must be unique in combination with the Identifier and TeamID. Name string `db:"name" json:"name"` - // Category is the category of the declaration, at the moment we - // only support configurations and activations. - Category MDMAppleDeclarationCategory `db:"category"` - // RawJSON is the raw JSON content of the declaration RawJSON json.RawMessage `db:"raw_json" json:"-"` @@ -604,7 +583,6 @@ var ForbiddenDeclTypes = map[string]struct{}{ "com.apple.configuration.account.google": {}, "com.apple.configuration.account.ldap": {}, "com.apple.configuration.account.mail": {}, - "com.apple.configuration.management.test": {}, "com.apple.configuration.screensharing.connection": {}, "com.apple.configuration.security.certificate": {}, "com.apple.configuration.security.identity": {}, @@ -629,7 +607,7 @@ func (r *MDMAppleRawDeclaration) ValidateUserProvided() error { return NewInvalidArgumentError(r.Type, "Declaration profile can’t include status subscription type. To get host’s vitals, please use queries and policies.") } - if !strings.HasPrefix(r.Type, string(MDMAppleDeclarativeConfiguration)) { + if !strings.HasPrefix(r.Type, "com.apple.configuration") { return NewInvalidArgumentError(r.Type, "Only configuration declarations (com.apple.configuration) are supported.") } @@ -671,12 +649,15 @@ type MDMAppleHostDeclaration struct { // Detail contains any messages that must be surfaced to the user, // either by the MDM protocol or the Fleet server. Detail string `db:"detail" json:"detail"` + + // Checksum contains the MD5 checksum of the declaration JSON uploaded + // by the IT admin. Fleet uses this value as the ServerToken. + Checksum string `db:"checksum" json:"-"` } func NewMDMAppleDeclaration(raw []byte, teamID *uint, name string, declType, ident string) *MDMAppleDeclaration { var decl MDMAppleDeclaration - decl.Category = MDMAppleDeclarationCategory(strings.Join(strings.Split(declType, ".")[:3], ".")) decl.Identifier = ident decl.Name = name decl.RawJSON = raw @@ -733,7 +714,6 @@ type MDMAppleDDMManifest struct { // https://developer.apple.com/documentation/devicemanagement/declarationitemsresponse type MDMAppleDDMDeclarationItem struct { Identifier string `db:"identifier"` - Category string `db:"category"` ServerToken string `db:"checksum"` } diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index 5b7cac96c8..e19f7e0b9e 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -1161,7 +1161,16 @@ type Datastore interface { // MDMAppleDDMDeclarationItems returns the declaration items for the specified host UUID. MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID string) ([]MDMAppleDDMDeclarationItem, error) // MDMAppleDDMDeclarationPayload returns the declaration payload for the specified identifier and team. - MDMAppleDDMDeclarationsResponse(ctx context.Context, declarationType MDMAppleDeclarationCategory, identifier string, hostUUID string) (*MDMAppleDeclaration, error) + MDMAppleDDMDeclarationsResponse(ctx context.Context, identifier string, hostUUID string) (*MDMAppleDeclaration, error) + // MDMAppleGetHostsWithChangedDeclarations returns a + // MDMAppleHostDeclaration item for each (host x declaration) pair that + // needs an status change, this includes declarations to install and + // declarations to be removed. Those can be differentiated by the + // OperationType field on each struct. + MDMAppleGetHostsWithChangedDeclarations(ctx context.Context) ([]*MDMAppleHostDeclaration, error) + // MDMAppleBatchInsertHostDeclarations tracks the current status of all + // the host declarations provided. + MDMAppleBatchInsertHostDeclarations(ctx context.Context, changedDeclarations []*MDMAppleHostDeclaration) error /////////////////////////////////////////////////////////////////////////////// // Microsoft MDM diff --git a/server/mock/datastore_mock.go b/server/mock/datastore_mock.go index fdc59496dd..7ab7c83a00 100644 --- a/server/mock/datastore_mock.go +++ b/server/mock/datastore_mock.go @@ -762,7 +762,11 @@ type MDMAppleDDMDeclarationsTokenFunc func(ctx context.Context, hostUUID string) type MDMAppleDDMDeclarationItemsFunc func(ctx context.Context, hostUUID string) ([]fleet.MDMAppleDDMDeclarationItem, error) -type MDMAppleDDMDeclarationsResponseFunc func(ctx context.Context, declarationType fleet.MDMAppleDeclarationCategory, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) +type MDMAppleDDMDeclarationsResponseFunc func(ctx context.Context, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) + +type MDMAppleGetHostsWithChangedDeclarationsFunc func(ctx context.Context) ([]*fleet.MDMAppleHostDeclaration, error) + +type MDMAppleBatchInsertHostDeclarationsFunc func(ctx context.Context, changedDeclarations []*fleet.MDMAppleHostDeclaration) error type WSTEPStoreCertificateFunc func(ctx context.Context, name string, crt *x509.Certificate) error @@ -1988,6 +1992,12 @@ type DataStore struct { MDMAppleDDMDeclarationsResponseFunc MDMAppleDDMDeclarationsResponseFunc MDMAppleDDMDeclarationsResponseFuncInvoked bool + MDMAppleGetHostsWithChangedDeclarationsFunc MDMAppleGetHostsWithChangedDeclarationsFunc + MDMAppleGetHostsWithChangedDeclarationsFuncInvoked bool + + MDMAppleBatchInsertHostDeclarationsFunc MDMAppleBatchInsertHostDeclarationsFunc + MDMAppleBatchInsertHostDeclarationsFuncInvoked bool + WSTEPStoreCertificateFunc WSTEPStoreCertificateFunc WSTEPStoreCertificateFuncInvoked bool @@ -4751,11 +4761,25 @@ func (s *DataStore) MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID st return s.MDMAppleDDMDeclarationItemsFunc(ctx, hostUUID) } -func (s *DataStore) MDMAppleDDMDeclarationsResponse(ctx context.Context, declarationType fleet.MDMAppleDeclarationCategory, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) { +func (s *DataStore) MDMAppleDDMDeclarationsResponse(ctx context.Context, identifier string, hostUUID string) (*fleet.MDMAppleDeclaration, error) { s.mu.Lock() s.MDMAppleDDMDeclarationsResponseFuncInvoked = true s.mu.Unlock() - return s.MDMAppleDDMDeclarationsResponseFunc(ctx, declarationType, identifier, hostUUID) + return s.MDMAppleDDMDeclarationsResponseFunc(ctx, identifier, hostUUID) +} + +func (s *DataStore) MDMAppleGetHostsWithChangedDeclarations(ctx context.Context) ([]*fleet.MDMAppleHostDeclaration, error) { + s.mu.Lock() + s.MDMAppleGetHostsWithChangedDeclarationsFuncInvoked = true + s.mu.Unlock() + return s.MDMAppleGetHostsWithChangedDeclarationsFunc(ctx) +} + +func (s *DataStore) MDMAppleBatchInsertHostDeclarations(ctx context.Context, changedDeclarations []*fleet.MDMAppleHostDeclaration) error { + s.mu.Lock() + s.MDMAppleBatchInsertHostDeclarationsFuncInvoked = true + s.mu.Unlock() + return s.MDMAppleBatchInsertHostDeclarationsFunc(ctx, changedDeclarations) } func (s *DataStore) WSTEPStoreCertificate(ctx context.Context, name string, crt *x509.Certificate) error { diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index b67796b33e..e64311c957 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -2710,6 +2710,55 @@ func ensureFleetdConfig(ctx context.Context, ds fleet.Datastore, logger kitlog.L return nil } +func ReconcileAppleDeclarations( + ctx context.Context, + ds fleet.Datastore, + commander *apple_mdm.MDMAppleCommander, + logger kitlog.Logger, +) error { + // once all the declarations are in place, compute the desired state + // and find which hosts need a DDM sync. + changedDeclarations, err := ds.MDMAppleGetHostsWithChangedDeclarations(ctx) + if err != nil { + return ctxerr.Wrap(ctx, err, "find hosts with changed declarations") + } + + if len(changedDeclarations) == 0 { + logger.Log("msg", "no hosts with changed declarations") + return nil + } + + // a host might have more than one declaration to sync, we do this to + // collect unique host UUIDs in order to send a single command to each + // host in the next step + uuidMap := map[string]struct{}{} + for _, d := range changedDeclarations { + uuidMap[d.HostUUID] = struct{}{} + } + uuids := make([]string, 0, len(uuidMap)) + for uuid := range uuidMap { + uuids = append(uuids, uuid) + } + + // mark the host declarations as pending, this serves two purposes: + // + // - support the APIs/methods that track host status (summaries, filters, etc) + // + // - support the DDM endpoints, which use data from the + // `host_mdm_apple_declarations` table to compute which declarations to + // serve + if err := ds.MDMAppleBatchInsertHostDeclarations(ctx, changedDeclarations); err != nil { + return ctxerr.Wrap(ctx, err, "batch insert mdm apple host declarations") + } + + // send a DeclarativeManagement command to start a sync + if err := commander.DeclarativeManagement(ctx, uuids, uuid.NewString()); err != nil { + return ctxerr.Wrap(ctx, err, "issuing DeclarativeManagement command") + } + + return nil +} + func ReconcileAppleProfiles( ctx context.Context, ds fleet.Datastore, @@ -3233,16 +3282,11 @@ func (svc *MDMAppleDDMService) handleDeclarationItems(ctx context.Context, hostU activations := []fleet.MDMAppleDDMManifest{} configurations := []fleet.MDMAppleDDMManifest{} for _, d := range di { - manifest := fleet.MDMAppleDDMManifest{Identifier: d.Identifier, ServerToken: d.ServerToken} - switch d.Category { - case string(fleet.MDMAppleDeclarativeActivation): - activations = append(activations, manifest) - case string(fleet.MDMAppleDeclarativeConfiguration): - configurations = append(configurations, manifest) - default: - level.Debug(svc.logger).Log("msg", "unrecognized declaration category", "category", d.Category) - return nil, ctxerr.New(ctx, "unrecognized declaration category") - } + configurations = append(configurations, fleet.MDMAppleDDMManifest(d)) + activations = append(activations, fleet.MDMAppleDDMManifest{ + Identifier: fmt.Sprintf("%s.activation", d.Identifier), + ServerToken: d.ServerToken, + }) } // TODO: Look for ways to optimize the declaration item query so that we don't have to get the declarations token separately. @@ -3274,11 +3318,43 @@ func (svc *MDMAppleDDMService) handleDeclarationsResponse(ctx context.Context, e } level.Debug(svc.logger).Log("msg", "parsed declarations request", "type", parts[1], "identifier", parts[2]) - d, err := svc.ds.MDMAppleDDMDeclarationsResponse( - ctx, fleet.MDMAppleDeclarationCategory("com.apple."+parts[1]), - parts[2], - hostUUID, - ) + switch parts[1] { + case "activation": + return svc.handleActivationDeclaration(ctx, parts, hostUUID) + case "configuration": + return svc.handleConfigurationDeclaration(ctx, parts, hostUUID) + default: + return nil, newNotFoundError() + } +} + +func (svc *MDMAppleDDMService) handleActivationDeclaration(ctx context.Context, parts []string, hostUUID string) ([]byte, error) { + references := strings.TrimSuffix(parts[2], ".activation") + + // ensure the declaration for the requested activation stil exists + d, err := svc.ds.MDMAppleDDMDeclarationsResponse(ctx, references, hostUUID) + if err != nil { + if fleet.IsNotFound(err) { + return nil, nano_service.NewHTTPStatusError(http.StatusNotFound, err) + } + return nil, ctxerr.Wrap(ctx, err, "getting linked configuration for activation declaration") + } + + response := fmt.Sprintf(` +{ + "Identifier": "%s", + "Payload": { + "StandardConfigurations": ["%s"] + }, + "ServerToken": "%s", + "Type": "com.apple.activation.simple" +}`, parts[2], references, d.Checksum) + + return []byte(response), nil +} + +func (svc *MDMAppleDDMService) handleConfigurationDeclaration(ctx context.Context, parts []string, hostUUID string) ([]byte, error) { + d, err := svc.ds.MDMAppleDDMDeclarationsResponse(ctx, parts[2], hostUUID) if err != nil { if fleet.IsNotFound(err) { return nil, nano_service.NewHTTPStatusError(http.StatusNotFound, err) @@ -3286,12 +3362,6 @@ func (svc *MDMAppleDDMService) handleDeclarationsResponse(ctx context.Context, e return nil, ctxerr.Wrap(ctx, err, "getting declaration response") } - // unmarshall into a temporary map in order to add the token. - // we do this at this stage because tokens are purely managed by Fleet, - // and we don't want to store a modified version of what's provided by - // the IT admin. - // - // This mimics what we do for CommandUUID, but can be revisited. var tempd map[string]any if err := json.Unmarshal(d.RawJSON, &tempd); err != nil { return nil, ctxerr.Wrap(ctx, err, "unmarshaling stored declaration") diff --git a/server/service/integration_mdm_test.go b/server/service/integration_mdm_test.go index fdf4f12af5..62039380ad 100644 --- a/server/service/integration_mdm_test.go +++ b/server/service/integration_mdm_test.go @@ -2755,9 +2755,14 @@ func (s *integrationMDMTestSuite) TestAppleMDMDeviceEnrollment() { mdmDeviceA := mdmtest.NewTestMDMClientAppleDirect(mdmEnrollInfo) err := mdmDeviceA.Enroll() require.NoError(t, err) + s.lastActivityOfTypeMatches(fleet.ActivityTypeMDMEnrolled{}.ActivityName(), + fmt.Sprintf(`{"host_serial": "%s", "host_display_name": "%s (%s)", "installed_from_dep": false, "mdm_platform": "apple"}`, mdmDeviceA.SerialNumber, mdmDeviceA.Model, mdmDeviceA.SerialNumber), 0) + mdmDeviceB := mdmtest.NewTestMDMClientAppleDirect(mdmEnrollInfo) err = mdmDeviceB.Enroll() require.NoError(t, err) + s.lastActivityOfTypeMatches(fleet.ActivityTypeMDMEnrolled{}.ActivityName(), + fmt.Sprintf(`{"host_serial": "%s", "host_display_name": "%s (%s)", "installed_from_dep": false, "mdm_platform": "apple"}`, mdmDeviceB.SerialNumber, mdmDeviceB.Model, mdmDeviceB.SerialNumber), 0) // Find the ID of Fleet's MDM solution var mdmID uint @@ -2786,23 +2791,6 @@ func (s *integrationMDMTestSuite) TestAppleMDMDeviceEnrollment() { } } - // Activities are generated for each device - activities := listActivitiesResponse{} - s.DoJSON("GET", "/api/latest/fleet/activities", nil, http.StatusOK, &activities, "order_key", "created_at") - require.GreaterOrEqual(t, len(activities.Activities), 2) - - details := []*json.RawMessage{} - for _, activity := range activities.Activities { - if activity.Type == "mdm_enrolled" { - require.Nil(t, activity.ActorID) - require.Nil(t, activity.ActorFullName) - details = append(details, activity.Details) - } - } - require.Len(t, details, 2) - require.JSONEq(t, fmt.Sprintf(`{"host_serial": "%s", "host_display_name": "%s (%s)", "installed_from_dep": false, "mdm_platform": "apple"}`, mdmDeviceA.SerialNumber, mdmDeviceA.Model, mdmDeviceA.SerialNumber), string(*details[len(details)-2])) - require.JSONEq(t, fmt.Sprintf(`{"host_serial": "%s", "host_display_name": "%s (%s)", "installed_from_dep": false, "mdm_platform": "apple"}`, mdmDeviceB.SerialNumber, mdmDeviceB.Model, mdmDeviceB.SerialNumber), string(*details[len(details)-1])) - // set an enroll secret var applyResp applyEnrollSecretSpecResponse s.DoJSON("POST", "/api/latest/fleet/spec/enroll_secret", applyEnrollSecretSpecRequest{ @@ -2839,7 +2827,7 @@ func (s *integrationMDMTestSuite) TestAppleMDMDeviceEnrollment() { require.NoError(t, err) // An activity is created - activities = listActivitiesResponse{} + activities := listActivitiesResponse{} s.DoJSON("GET", "/api/latest/fleet/activities", nil, http.StatusOK, &activities) found := false @@ -2848,7 +2836,6 @@ func (s *integrationMDMTestSuite) TestAppleMDMDeviceEnrollment() { found = true require.Nil(t, activity.ActorID) require.Nil(t, activity.ActorFullName) - details = append(details, activity.Details) require.JSONEq(t, fmt.Sprintf(`{"host_serial": "%s", "host_display_name": "%s (%s)", "installed_from_dep": false}`, mdmDeviceA.SerialNumber, mdmDeviceA.Model, mdmDeviceA.SerialNumber), string(*activity.Details)) } } @@ -9472,9 +9459,9 @@ func (s *integrationMDMTestSuite) TestMDMConfigProfileCRUD() { var profileLabels []fleet.ConfigurationProfileLabel mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { stmt := ` - SELECT COALESCE(apple_profile_uuid, windows_profile_uuid) as profile_uuid, label_name, label_id + SELECT COALESCE(apple_profile_uuid, windows_profile_uuid) as profile_uuid, label_name, COALESCE(label_id, 0) as label_id FROM mdm_configuration_profile_labels - UNION SELECT apple_declaration_uuid as profile_uuid, label_name, label_id + UNION SELECT apple_declaration_uuid as profile_uuid, label_name, COALESCE(label_id, 0) as label_id FROM mdm_declaration_labels ORDER BY profile_uuid, label_name;` return sqlx.SelectContext(context.Background(), q, &profileLabels, stmt) }) @@ -12783,12 +12770,11 @@ INSERT INTO mdm_apple_declarations ( team_id, identifier, name, - category, raw_json, checksum, created_at, uploaded_at -) VALUES (?,?,?,?,?,?,UNHEX(?),?,?)` +) VALUES (?,?,?,?,?,UNHEX(?),?,?)` mysql.ExecAdhocSQL(t, s.ds, func(q sqlx.ExtContext) error { _, err := q.ExecContext(context.Background(), stmt, @@ -12796,7 +12782,6 @@ INSERT INTO mdm_apple_declarations ( decl.TeamID, decl.Identifier, decl.Name, - decl.Category, decl.RawJSON, calcChecksum(decl.RawJSON), decl.CreatedAt, @@ -12839,7 +12824,6 @@ INSERT INTO host_mdm_apple_declarations ( TeamID: ptr.Uint(0), Identifier: "com.example", Name: "Example", - Category: fleet.MDMAppleDeclarativeConfiguration, RawJSON: json.RawMessage(`{ "Type": "com.apple.configuration.declaration-items.test", "Payload": {"foo":"bar"}, @@ -12904,7 +12888,6 @@ INSERT INTO host_mdm_apple_declarations ( require.NoError(t, json.NewDecoder(r.Body).Decode(&gotParsed)) require.EqualValues(t, wantParsed.Payload, gotParsed.Payload) require.Equal(t, calcChecksum(expected.RawJSON), gotParsed.ServerToken) - require.Contains(t, gotParsed.Type, expected.Category) require.Equal(t, expected.Identifier, gotParsed.Identifier) // t.Logf("decoded: %+v", gotParsed) } @@ -12917,7 +12900,8 @@ INSERT INTO host_mdm_apple_declarations ( checkDeclarationItemsResp := func(t *testing.T, r fleet.MDMAppleDDMDeclarationItemsResponse, expectedDeclTok string, expectedDeclsByChecksum map[string]fleet.MDMAppleDeclaration) { require.Equal(t, expectedDeclTok, r.DeclarationsToken) - require.Empty(t, r.Declarations.Activations) + // TODO(roberto): better assertions + require.NotEmpty(t, r.Declarations.Activations) require.Empty(t, r.Declarations.Assets) require.Empty(t, r.Declarations.Management) require.Len(t, r.Declarations.Configurations, len(expectedDeclsByChecksum)) @@ -12944,7 +12928,6 @@ INSERT INTO host_mdm_apple_declarations ( TeamID: ptr.Uint(0), Identifier: "com.example2", Name: "Example2", - Category: fleet.MDMAppleDeclarativeConfiguration, RawJSON: json.RawMessage(`{ "Type": "com.apple.configuration.declaration-items.test", "Payload": {"foo":"baz"}, @@ -12975,7 +12958,6 @@ INSERT INTO host_mdm_apple_declarations ( TeamID: ptr.Uint(0), Identifier: "com.example3", Name: "Example3", - Category: fleet.MDMAppleDeclarativeConfiguration, RawJSON: json.RawMessage(`{ "Type": "com.apple.configuration.declaration-items.test", "Payload": {"foo":"bang"}, @@ -13017,7 +12999,6 @@ INSERT INTO host_mdm_apple_declarations ( TeamID: ptr.Uint(0), Identifier: "com.example4", Name: "Example4", - Category: fleet.MDMAppleDeclarativeConfiguration, RawJSON: json.RawMessage(`{ "Type": "com.apple.configuration.test", "Payload": {"foo":"bar"}, @@ -13045,3 +13026,253 @@ INSERT INTO host_mdm_apple_declarations ( assertDeclarationResponse(r, want) }) } + +func (s *integrationMDMTestSuite) TestAppleDDMReconciliation() { + t := s.T() + ctx := context.Background() + // TODO: use config logger or take into account FLEET_INTEGRATION_TESTS_DISABLE_LOG + logger := kitlog.NewJSONLogger(os.Stdout) + + // TODO: use endpoints once those are available. + addDeclaration := func(identifier string, teamID uint) { + stmt := ` + INSERT INTO mdm_apple_declarations + (declaration_uuid, team_id, identifier, name, raw_json, checksum) + VALUES + (UUID(), ?, ?, UUID(), ?, HEX(MD5(raw_json)) )` + mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error { + _, err := tx.ExecContext(ctx, stmt, teamID, identifier, declarationForTest(identifier)) + return err + }) + } + + deleteDeclaration := func(identifier string, teamID uint) { + mysql.ExecAdhocSQL(t, s.ds, func(tx sqlx.ExtContext) error { + _, err := tx.ExecContext(ctx, "DELETE FROM mdm_apple_declarations WHERE team_id = ? AND identifier = ?", teamID, identifier) + return err + }) + } + + // create a team + teamName := t.Name() + "team1" + team := &fleet.Team{ + Name: teamName, + } + var createTeamResp teamResponse + s.DoJSON("POST", "/api/latest/fleet/teams", team, http.StatusOK, &createTeamResp) + require.NotZero(t, createTeamResp.Team.ID) + team = createTeamResp.Team + + t.Cleanup(func() { + // delete declarations to not affect other tests + deleteDeclaration("I2", 0) + deleteDeclaration("I1", team.ID) + deleteDeclaration("I2", team.ID) + deleteDeclaration("I3", team.ID) + }) + + checkNoCommands := func(d *mdmtest.TestAppleMDMClient) { + cmd, err := d.Idle() + require.NoError(t, err) + require.Nil(t, cmd) + } + + checkDDMSync := func(d *mdmtest.TestAppleMDMClient) { + cmd, err := d.Idle() + require.NoError(t, err) + require.NotNil(t, cmd) + require.Equal(t, "DeclarativeManagement", cmd.Command.RequestType) + cmd, err = d.Acknowledge(cmd.CommandUUID) + require.NoError(t, err) + require.Nil(t, cmd) + _, err = d.DeclarativeManagement("tokens") + require.NoError(t, err) + } + + // create a windows host + _, err := s.ds.NewHost(context.Background(), &fleet.Host{ + ID: 1, + OsqueryHostID: ptr.String("non-macos-host"), + NodeKey: ptr.String("non-macos-host"), + UUID: uuid.New().String(), + Hostname: fmt.Sprintf("%sfoo.local.non.macos", t.Name()), + Platform: "windows", + }) + require.NoError(t, err) + + // create a windows host that's enrolled in MDM + _, _ = createWindowsHostThenEnrollMDM(s.ds, s.server.URL, t) + + // create a linux host + _, err = s.ds.NewHost(context.Background(), &fleet.Host{ + ID: 2, + OsqueryHostID: ptr.String("linux-host"), + NodeKey: ptr.String("linux-host"), + UUID: uuid.New().String(), + Hostname: fmt.Sprintf("%sfoo.local.linux", t.Name()), + Platform: "linux", + }) + require.NoError(t, err) + + // create a host that's not enrolled into MDM + _, err = s.ds.NewHost(context.Background(), &fleet.Host{ + ID: 2, + OsqueryHostID: ptr.String("not-mdm-enrolled"), + NodeKey: ptr.String("not-mdm-enrolled"), + UUID: uuid.New().String(), + Hostname: fmt.Sprintf("%sfoo.local.not.enrolled", t.Name()), + Platform: "darwin", + }) + require.NoError(t, err) + + // create a host and then enroll in MDM. + mdmHost, device := createHostThenEnrollMDM(s.ds, s.server.URL, t) + + // trigger the reconciler, no error + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + + // declarativeManagement command is not sent. + checkNoCommands(device) + + // add global declarations + addDeclaration("I1", 0) + addDeclaration("I2", 0) + + // reconcile again, this time new declarations were added + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + + // TODO: check command is pending + + // declarativeManagement command is sent + checkDDMSync(device) + + // reconcile again, commands for the uploaded declarations are already sent + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // no new commands are sent + checkNoCommands(device) + + // delete a declaration + deleteDeclaration("I1", 0) + // reconcile again + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // a DDM sync is triggered + checkDDMSync(device) + + // add a new host + _, deviceTwo := createHostThenEnrollMDM(s.ds, s.server.URL, t) + // reconcile again + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // DDM sync is triggered only for the new host + checkNoCommands(device) + checkDDMSync(deviceTwo) + + // add device to the team + s.Do("POST", "/api/v1/fleet/hosts/transfer", + addHostsToTeamRequest{TeamID: &team.ID, HostIDs: []uint{mdmHost.ID}}, http.StatusOK) + + // reconcile + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + + // DDM sync is triggered only for the transferred host + // because the team doesn't have any declarations + checkDDMSync(device) + checkNoCommands(deviceTwo) + + // reconcile + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // nobody receives commands this time + checkNoCommands(device) + checkNoCommands(deviceTwo) + + // add declarations to the team + addDeclaration("I1", team.ID) + addDeclaration("I2", team.ID) + + // reconcile + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // DDM sync is triggered for the host in the team + checkDDMSync(device) + checkNoCommands(deviceTwo) + + // add a new host, this one belongs to the team + mdmHostThree, deviceThree := createHostThenEnrollMDM(s.ds, s.server.URL, t) + s.Do("POST", "/api/v1/fleet/hosts/transfer", + addHostsToTeamRequest{TeamID: &team.ID, HostIDs: []uint{mdmHostThree.ID}}, http.StatusOK) + + // reconcile + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // DDM sync is triggered only for the new host + checkNoCommands(device) + checkNoCommands(deviceTwo) + checkDDMSync(deviceThree) + + // no new commands after another reconciliation + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + checkNoCommands(device) + checkNoCommands(deviceTwo) + checkNoCommands(deviceThree) + + // TODO: use proper APIs for this + // add a new label + label declaration + addDeclaration("I3", team.ID) + label, err := s.ds.NewLabel(ctx, &fleet.Label{Name: t.Name(), Query: "select 1;"}) + require.NoError(t, err) + // update label with host membership + mysql.ExecAdhocSQL( + t, s.ds, func(db sqlx.ExtContext) error { + _, err := db.ExecContext( + context.Background(), + "INSERT IGNORE INTO label_membership (host_id, label_id) VALUES (?, ?)", + mdmHostThree.ID, + label.ID, + ) + return err + }, + ) + + // update declaration <-> label mapping + mysql.ExecAdhocSQL( + t, s.ds, func(db sqlx.ExtContext) error { + _, err := db.ExecContext( + context.Background(), + `INSERT INTO + mdm_declaration_labels (apple_declaration_uuid, label_name, label_id) + VALUES ((SELECT declaration_uuid FROM mdm_apple_declarations WHERE team_id = ? and identifier = ?), ?, ?)`, + team.ID, + "I3", + label.Name, + label.ID, + ) + return err + }, + ) + + // reconcile + err = ReconcileAppleDeclarations(ctx, s.ds, s.mdmCommander, logger) + require.NoError(t, err) + // DDM sync is triggered only for the host with the label + checkNoCommands(device) + checkNoCommands(deviceTwo) + checkDDMSync(deviceThree) +} + +func declarationForTest(identifier string) []byte { + return []byte(fmt.Sprintf(` +{ + "Type": "com.apple.configuration.management.test", + "Payload": { + "Echo": "foo" + }, + "Identifier": "%s" +}`, identifier)) +}