Surface proxied Windows SCEP certificate failures (#45550) (#48842)

Windows configuration profiles that Fleet proxies SCEP for previously
reported "verified" as soon as the device acknowledged the SyncML Exec
command, even when the asynchronous SCEP exchange later failed and no
certificate was ever issued.

- Proxied SCEP profiles (custom SCEP proxy, NDES) now move to
"verifying" on the device ACK and only reach "verified" once Fleet
observes the matching certificate on the host, keyed by the renewal-ID
marker (fleet-<profile_uuid>) in the certificate CN/OU.
- When Fleet's SCEP proxy observes an upstream CA error during
PKIOperation, it marks the profile "failed" with a detail naming the
operation and upstream status. If the device's own retry later succeeds,
the observed certificate flips the profile to "verified".
- Unconfirmed profiles stay "verifying" (offline host, agent that cannot
enumerate certificates, empty store, or a user-scoped profile before the
user logs in); absence is never treated as failure.

<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #45550 

Demo: https://www.youtube.com/watch?v=WNGuFdeBmzA
Docs: https://github.com/fleetdm/fleet/pull/48933/changes

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

- [x] Timeouts are implemented and retries are limited to avoid infinite
loops

## Testing

- [x] Added/updated automated tests
- [x] QA'd all new/changed functionality manually

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Summary by CodeRabbit

* **New Features**
* Added Windows SCEP failure tracking with clearer, categorized detail
when upstream operations fail.
* Added reconciliation backstops for “stuck” proxied SCEP profiles,
including automatic recovery to verified when the expected certificate
is observed.

* **Bug Fixes**
* Prevented proxied Windows SCEP installs from being marked “verified”
until matching certificate evidence arrives.
* Improved classification and persistence behavior for timeouts,
connection/DNS issues, and HTTP error responses without disturbing
existing retry state.

* **Tests**
* Expanded Windows SCEP scenarios to cover reconciliation, skipping
conditions, and error classification.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Victor Lyuboslavsky
2026-07-09 07:38:47 +01:00
committed by GitHub
parent 00d7f4b63a
commit e1094096af
9 changed files with 719 additions and 1 deletions
@@ -0,0 +1 @@
- Windows configuration profiles that use a Fleet-proxied SCEP certificate (custom SCEP proxy, NDES, or Smallstep) now report "Verified" only after Fleet observes the issued certificate on the host, instead of reporting "Verified" as soon as the host acknowledged the profile. They report "Failed" when the SCEP proxy request returns an upstream error, or when the certificate is still missing from the host an hour after delivery (once Fleet can confirm the certificate's store was readable).
+74
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"io"
"log/slog"
"net"
"net/http"
"net/url"
"regexp"
@@ -40,6 +41,9 @@ const (
MessageSCEPProxyNotConfigured = "SCEP proxy is not configured"
NDESChallengeInvalidAfter = 57 * time.Minute
SmallstepChallengeInvalidAfter = 4 * time.Minute
// windowsSCEPFailureWriteTimeout bounds the detached write that records a Windows SCEP profile failure, so it can
// still persist when the originating request context is already at its deadline.
windowsSCEPFailureWriteTimeout = 5 * time.Second
)
// decodeHTMLResponse decodes HTTP response body to a string, handling various encodings.
@@ -268,12 +272,82 @@ func (svc *scepProxyService) PKIOperation(ctx context.Context, data []byte, iden
}
res, err := client.PKIOperation(ctx, data)
if err != nil {
svc.recordWindowsSCEPProxyFailure(ctx, identifier, "PKIOperation", err)
return res, ctxerr.Wrapf(ctx, err,
"Could not do PKIOperation on SCEP server %s", scepURL)
}
return res, nil
}
// recordWindowsSCEPProxyFailure marks a Windows SCEP profile "failed" when the proxy observes a per-profile upstream
// error.
func (svc *scepProxyService) recordWindowsSCEPProxyFailure(ctx context.Context, identifier, operation string, upstreamErr error) {
// A canceled request context (device disconnected mid-exchange, reverse proxy aborted, or server shutting down) is
// not an upstream CA failure and must not mark the profile failed. Genuine upstream timeouts arrive as
// context.DeadlineExceeded / net timeouts and are still surfaced.
if errors.Is(upstreamErr, context.Canceled) || errors.Is(ctx.Err(), context.Canceled) {
return
}
hostUUID, profileUUID, ok := parseHostAndProfileFromSCEPIdentifier(identifier)
if !ok || !strings.HasPrefix(profileUUID, fleet.MDMWindowsProfileUUIDPrefix) {
return
}
detail := fmt.Sprintf("SCEP %s failed: %s", operation, classifySCEPProxyError(upstreamErr))
// Detach the write from the request's deadline/cancellation: an upstream timeout often leaves the request context
// already at its deadline, and we must still persist the failure we observed. WithoutCancel preserves request
// values (tracing, etc.) while dropping the deadline; the write gets its own short timeout.
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), windowsSCEPFailureWriteTimeout)
defer cancel()
if err := svc.ds.SetMDMWindowsHostProfileFailed(writeCtx, hostUUID, profileUUID, detail); err != nil {
svc.debugLogger.ErrorContext(ctx, "recording Windows SCEP proxy failure",
"host_uuid", hostUUID, "profile_uuid", profileUUID, "err", err)
ctxerr.Handle(ctx, err)
}
}
// parseHostAndProfileFromSCEPIdentifier extracts the host and profile UUIDs from the SCEP proxy identifier
// ("hostUUID,profileUUID,caName,challenge"). It intentionally does no validation beyond the two leading fields; full
// validation lives in validateIdentifier.
func parseHostAndProfileFromSCEPIdentifier(identifier string) (hostUUID, profileUUID string, ok bool) {
parsed, err := url.PathUnescape(identifier)
if err != nil {
return "", "", false
}
parts := strings.Split(parsed, ",")
if len(parts) < 2 || parts[0] == "" || parts[1] == "" {
return "", "", false
}
return parts[0], parts[1], true
}
var scepProxyHTTPStatusRegex = regexp.MustCompile(`status (\d{3})`)
// classifySCEPProxyError renders a short, stable, human-readable reason for a Windows profile's failure detail.
func classifySCEPProxyError(err error) string {
if err == nil {
return "unknown error"
}
if errors.Is(err, context.DeadlineExceeded) {
return "timeout"
}
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
return "timeout"
}
msg := err.Error()
if m := scepProxyHTTPStatusRegex.FindStringSubmatch(msg); m != nil {
return "HTTP " + m[1]
}
switch {
case strings.Contains(msg, "connection refused"):
return "connection refused"
case strings.Contains(msg, "no such host"):
return "DNS resolution error"
default:
return "upstream error"
}
}
func (svc *scepProxyService) validateIdentifier(ctx context.Context, identifier string, checkChallenge bool) (string,
error,
) {
+86
View File
@@ -5,6 +5,7 @@ import (
"database/sql"
"encoding/binary"
"errors"
"fmt"
"log/slog"
"net/http"
"net/http/httptest"
@@ -1164,3 +1165,88 @@ func TestValidateIdentifier(t *testing.T) {
ds.ConsumeChallengeFuncInvoked = false
})
}
func TestClassifySCEPProxyError(t *testing.T) {
for _, tc := range []struct {
name string
err error
want string
}{
{"nil", nil, "unknown error"},
{"deadline exceeded", context.DeadlineExceeded, "timeout"},
{"wrapped deadline", fmt.Errorf("doing PKIOperation: %w", context.DeadlineExceeded), "timeout"},
{"net timeout", os.ErrDeadlineExceeded, "timeout"}, // implements net.Error with Timeout() == true
{"http 500", errors.New("http request failed with status 500 Internal Server Error, msg: boom"), "HTTP 500"},
{"http 403", errors.New("http request failed with status 403 Forbidden, msg: denied"), "HTTP 403"},
{"connection refused", errors.New("dial tcp 10.0.0.1:80: connect: connection refused"), "connection refused"},
{"dns", errors.New("dial tcp: lookup ca.invalid: no such host"), "DNS resolution error"},
{"generic", errors.New("something unexpected happened"), "upstream error"},
} {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.want, classifySCEPProxyError(tc.err))
})
}
}
func TestRecordWindowsSCEPProxyFailure(t *testing.T) {
logger := slog.New(slog.DiscardHandler)
newSvc := func(ds *mock.DataStore) *scepProxyService {
return &scepProxyService{ds: ds, debugLogger: logger}
}
const winID = "host-uuid,w-profile-uuid,ca,challenge"
upstreamErr := errors.New("http request failed with status 500 Internal Server Error, msg: boom")
t.Run("records a real upstream error for a Windows profile", func(t *testing.T) {
ds := new(mock.DataStore)
var gotDetail string
ds.SetMDMWindowsHostProfileFailedFunc = func(_ context.Context, hostUUID, profileUUID, detail string) error {
assert.Equal(t, "host-uuid", hostUUID)
assert.Equal(t, "w-profile-uuid", profileUUID)
gotDetail = detail
return nil
}
newSvc(ds).recordWindowsSCEPProxyFailure(context.Background(), winID, "PKIOperation", upstreamErr)
require.True(t, ds.SetMDMWindowsHostProfileFailedFuncInvoked)
assert.Equal(t, "SCEP PKIOperation failed: HTTP 500", gotDetail)
})
t.Run("records with a live context even when the request deadline is exceeded", func(t *testing.T) {
ds := new(mock.DataStore)
ds.SetMDMWindowsHostProfileFailedFunc = func(ctx context.Context, _, _, _ string) error {
// The write must be detached from the expired request context (WithoutCancel), or it would fail to persist
// the failure we just observed. This assertion is what actually guards that detach.
assert.NoError(t, ctx.Err())
return nil
}
// Deadline already in the past (as after an upstream timeout): ctx.Err() is DeadlineExceeded, which must NOT
// skip recording - only true cancellation does.
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Hour))
defer cancel()
newSvc(ds).recordWindowsSCEPProxyFailure(ctx, winID, "PKIOperation", upstreamErr)
require.True(t, ds.SetMDMWindowsHostProfileFailedFuncInvoked)
})
// Cases where the failure must NOT be recorded. t.Fatal in the mock is the assertion.
canceledCtx, cancelFn := context.WithCancel(context.Background())
cancelFn()
for _, tc := range []struct {
name string
ctx context.Context
id string
err error
}{
{"a canceled upstream error", context.Background(), winID, context.Canceled},
{"a canceled request context", canceledCtx, winID, upstreamErr},
{"a non-Windows profile", context.Background(), "host-uuid,a-apple-profile,ca,challenge", upstreamErr},
{"a malformed identifier", context.Background(), "garbage-without-commas", upstreamErr},
} {
t.Run("skips "+tc.name, func(t *testing.T) {
ds := new(mock.DataStore)
ds.SetMDMWindowsHostProfileFailedFunc = func(context.Context, string, string, string) error {
t.Fatalf("must not record a failure for %s", tc.name)
return nil
}
newSvc(ds).recordWindowsSCEPProxyFailure(tc.ctx, tc.id, "PKIOperation", tc.err)
})
}
}
+132
View File
@@ -367,6 +367,13 @@ func (ds *Datastore) UpdateHostCertificates(ctx context.Context, hostID uint, ho
}
}
// Whether osquery could read at least one user's certificate store in this report. SINGLE-USER ASSUMPTION: we treat
// "any user cert observed" as "the target user's store was readable", which holds when the device has one primary
// user.
anyUserCertObserved := slices.ContainsFunc(certs, func(c *fleet.HostCertificateRecord) bool {
return c.Source == fleet.UserHostCertificate
})
return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
if err := insertHostCertsDB(ctx, tx, toInsert); err != nil {
return ctxerr.Wrap(ctx, err, "insert host certs")
@@ -409,10 +416,135 @@ func (ds *Datastore) UpdateHostCertificates(ctx context.Context, hostID uint, ho
if err := insertHostMDMManagedCertDB(ctx, tx, hostMDMManagedCertsToInsert); err != nil {
return ctxerr.Wrap(ctx, err, "insert host mdm managed cert rows")
}
// A proxied Windows SCEP profile sits in "verifying" until the certificate it requested is observed on the host.
// The managed-cert updates above set not_valid_after/serial when a reported cert matched the profile's
// renewal-ID marker, so a matched-and-valid managed-cert row is the signal that the certificate landed. Flip
// those profiles to "verified" (self-healing any that were "failed" from a proxy-observed error).
if err := verifyWindowsSCEPProfilesFromObservedCertsDB(ctx, tx, hostUUID); err != nil {
return ctxerr.Wrap(ctx, err, "verify windows scep profiles from observed certs")
}
// Backstop: a proxied Windows SCEP profile that never gets its certificate would otherwise sit in
// "verifying" forever. Once the grace period has elapsed and this report proves we could read the store
// where the certificate belongs but it isn't there, fail it. Runs after the flip above, so anything still
// "verifying" here has no observed certificate.
if err := failStuckWindowsSCEPProfilesDB(ctx, tx, hostUUID, anyUserCertObserved); err != nil {
return ctxerr.Wrap(ctx, err, "fail stuck windows scep profiles")
}
return nil
})
}
// windowsSCEPVerificationGracePeriod is how long a proxied Windows SCEP profile may stay in "verifying" (measured from
// the host's ACK of the profile, i.e. host_mdm_windows_profiles.updated_at) before an ingest that proves the relevant
// certificate store was readable, yet lacks the certificate, is treated as a failure.
const windowsSCEPVerificationGracePeriod = time.Hour
// windowsSCEPCertNotFoundDetail is the failure detail recorded when the verification backstop fires.
const windowsSCEPCertNotFoundDetail = "Fleet did not detect the SCEP certificate on the host after profile was delivered."
// failStuckWindowsSCEPProfilesDB is the verification backstop for proxied Windows SCEP profiles. It runs on certificate
// ingestion (so an offline host, or one whose agent can't enumerate certificates, never ingests and is never failed)
// and marks a profile "failed" only when we have positive evidence the certificate is missing:
//
// - Device-scoped profiles (SyncML uses the ./Device SCEP node): the LocalMachine store is always readable when
// osquery reports, so any ingest past the grace period with the certificate still absent is a genuine failure.
// - User-scoped profiles (SyncML uses the ./User SCEP node): the certificate lives in a user's store, which osquery
// can read only while that user is logged in. We fail only when this report includes at least one user
// certificate, proving a user store was readable. SINGLE-USER ASSUMPTION: Fleet does not track which user a
// ./User Windows profile targets, so we assume the device has one primary user.
func failStuckWindowsSCEPProfilesDB(ctx context.Context, tx sqlx.ExtContext, hostUUID string, anyUserCertObserved bool) error {
caTypes := fleet.ListCATypesWithRenewalIDSupport()
caTypeStrs := make([]string, 0, len(caTypes))
for _, t := range caTypes {
caTypeStrs = append(caTypeStrs, string(t))
}
graceSeconds := int(windowsSCEPVerificationGracePeriod.Seconds())
var query string
var args []any
if anyUserCertObserved {
// System and user scope observed. No need to inspect the profile's SyncML scope.
query = `
UPDATE host_mdm_windows_profiles hwmp
JOIN host_mdm_managed_certificates hmmc
ON hmmc.host_uuid = hwmp.host_uuid AND hmmc.profile_uuid = hwmp.profile_uuid
SET hwmp.status = ?, hwmp.detail = ?
WHERE hwmp.host_uuid = ?
AND hwmp.operation_type = ?
AND hwmp.status = ?
AND hmmc.type IN (?)
AND hwmp.updated_at < DATE_SUB(NOW(), INTERVAL ? SECOND)`
args = []any{
fleet.MDMDeliveryFailed, windowsSCEPCertNotFoundDetail, hostUUID, fleet.MDMOperationTypeInstall,
fleet.MDMDeliveryVerifying, caTypeStrs, graceSeconds,
}
} else {
// Only the LocalMachine store is provably readable this run. Restrict to device-scoped profiles (SyncML
// without a ./User SCEP node); a user-scoped certificate may just be waiting for its user to log in.
query = `
UPDATE host_mdm_windows_profiles hwmp
JOIN host_mdm_managed_certificates hmmc
ON hmmc.host_uuid = hwmp.host_uuid AND hmmc.profile_uuid = hwmp.profile_uuid
JOIN mdm_windows_configuration_profiles cp
ON cp.profile_uuid = hwmp.profile_uuid
SET hwmp.status = ?, hwmp.detail = ?
WHERE hwmp.host_uuid = ?
AND hwmp.operation_type = ?
AND hwmp.status = ?
AND hmmc.type IN (?)
AND hwmp.updated_at < DATE_SUB(NOW(), INTERVAL ? SECOND)
AND cp.syncml NOT LIKE ?`
args = []any{
fleet.MDMDeliveryFailed, windowsSCEPCertNotFoundDetail, hostUUID, fleet.MDMOperationTypeInstall,
fleet.MDMDeliveryVerifying, caTypeStrs, graceSeconds, "%/User/Vendor/MSFT/ClientCertificateInstall/SCEP%",
}
}
stmt, inArgs, err := sqlx.In(query, args...)
if err != nil {
return ctxerr.Wrap(ctx, err, "building windows scep backstop query")
}
if _, err := tx.ExecContext(ctx, stmt, inArgs...); err != nil {
return ctxerr.Wrap(ctx, err, "failing stuck windows scep profiles")
}
return nil
}
// verifyWindowsSCEPProfilesFromObservedCertsDB flips a host's proxied Windows SCEP install profiles from "verifying" or
// "failed" to "verified" once their managed-certificate row shows a certificate was observed (its serial/validity dates
// were populated by the renewal-ID matcher in UpdateHostCertificates). Current validity is intentionally NOT required:
// observing that the CA issued a certificate matching this profile's renewal-ID proves the enrollment succeeded, so we
// mark it verified regardless of the certificate's lifetime. A short-lived certificate that has since expired is a
// renewal concern (handled by RenewMDMManagedCertificates), not a verification failure.
func verifyWindowsSCEPProfilesFromObservedCertsDB(ctx context.Context, tx sqlx.ExtContext, hostUUID string) error {
caTypes := fleet.ListCATypesWithRenewalIDSupport()
caTypeStrs := make([]string, 0, len(caTypes))
for _, t := range caTypes {
caTypeStrs = append(caTypeStrs, string(t))
}
stmt, args, err := sqlx.In(`
UPDATE host_mdm_windows_profiles hwmp
JOIN host_mdm_managed_certificates hmmc
ON hmmc.host_uuid = hwmp.host_uuid AND hmmc.profile_uuid = hwmp.profile_uuid
SET hwmp.status = ?, hwmp.detail = ''
WHERE hwmp.host_uuid = ?
AND hwmp.operation_type = ?
AND hwmp.status IN (?, ?)
AND hmmc.type IN (?)
AND hmmc.not_valid_after IS NOT NULL`,
fleet.MDMDeliveryVerified, hostUUID, fleet.MDMOperationTypeInstall,
fleet.MDMDeliveryVerifying, fleet.MDMDeliveryFailed, caTypeStrs)
if err != nil {
return ctxerr.Wrap(ctx, err, "building windows scep verify query")
}
if _, err := tx.ExecContext(ctx, stmt, args...); err != nil {
return ctxerr.Wrap(ctx, err, "flipping windows scep profiles to verified")
}
return nil
}
// validateAndTruncateCertificateFields validates and truncates certificate string fields to match database schema constraints
func (ds *Datastore) validateAndTruncateCertificateFields(ctx context.Context, hostID uint, cert *fleet.HostCertificateRecord) {
// Field length limits based on schema
@@ -43,6 +43,7 @@ func TestHostCertificates(t *testing.T) {
{"Truncate long certificate fields", testTruncateLongCertificateFields},
{"Count matches main query", testListHostCertificatesCountMatches},
{"Sweep mdm certs for unenrolled hosts", testSoftDeleteMDMHostCertificatesForUnenrolledHosts},
{"Windows proxied SCEP profile verification", testWindowsSCEPProfileVerification},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
@@ -52,6 +53,309 @@ func TestHostCertificates(t *testing.T) {
}
}
func testWindowsSCEPProfileVerification(t *testing.T, ds *Datastore) {
ctx := t.Context()
mkHost := func(t *testing.T, suffix string) *fleet.Host {
t.Helper()
osqueryID := "wscep-osquery-" + suffix
nodeKey := "wscep-node-" + suffix
h, err := ds.NewHost(ctx, &fleet.Host{
DetailUpdatedAt: time.Now(),
LabelUpdatedAt: time.Now(),
PolicyUpdatedAt: time.Now(),
SeenTime: time.Now(),
OsqueryHostID: &osqueryID,
NodeKey: &nodeKey,
UUID: "wscep-host-" + suffix,
Hostname: "wscep-hostname-" + suffix,
})
require.NoError(t, err)
return h
}
upsertWinProfile := func(t *testing.T, h *fleet.Host, profileUUID, cmdUUID string, status fleet.MDMDeliveryStatus, retries int) {
t.Helper()
require.NoError(t, ds.BulkUpsertMDMWindowsHostProfiles(ctx, []*fleet.MDMWindowsBulkUpsertHostProfilePayload{{
ProfileUUID: profileUUID,
ProfileName: "p-" + profileUUID,
HostUUID: h.UUID,
CommandUUID: cmdUUID,
OperationType: fleet.MDMOperationTypeInstall,
Status: &status,
Checksum: []byte{1},
}}))
if retries > 0 {
_, err := ds.writer(ctx).ExecContext(ctx,
`UPDATE host_mdm_windows_profiles SET retries = ? WHERE host_uuid = ? AND profile_uuid = ?`, retries, h.UUID, profileUUID)
require.NoError(t, err)
}
}
upsertHMMC := func(t *testing.T, h *fleet.Host, profileUUID string, caType fleet.CAConfigAssetType, nvb, nva *time.Time) {
t.Helper()
hmmc := &fleet.MDMManagedCertificate{HostUUID: h.UUID, ProfileUUID: profileUUID, Type: caType, CAName: "ca-" + profileUUID}
if nva != nil {
hmmc.NotValidBefore = nvb
hmmc.NotValidAfter = nva
serial := "serial-" + profileUUID
hmmc.Serial = &serial
}
require.NoError(t, ds.BulkUpsertMDMManagedCertificates(ctx, []*fleet.MDMManagedCertificate{hmmc}))
}
getProfile := func(t *testing.T, h *fleet.Host, profileUUID string) (fleet.MDMDeliveryStatus, string, int) {
t.Helper()
var row struct {
Status fleet.MDMDeliveryStatus `db:"status"`
Detail string `db:"detail"`
Retries int `db:"retries"`
}
require.NoError(t, sqlx.GetContext(ctx, ds.reader(ctx), &row,
`SELECT status, detail, retries FROM host_mdm_windows_profiles WHERE host_uuid = ? AND profile_uuid = ?`, h.UUID, profileUUID))
return row.Status, row.Detail, row.Retries
}
// certWithRenewalID builds an ingestable host cert whose OU carries the profile's renewal-ID marker.
certWithRenewalID := func(t *testing.T, h *fleet.Host, renewalProfileUUID string, serial int64) *fleet.HostCertificateRecord {
t.Helper()
tmpl := &x509.Certificate{
Subject: pkix.Name{
CommonName: "device " + renewalProfileUUID,
Organization: []string{"Org"},
OrganizationalUnit: []string{"fleet-" + renewalProfileUUID},
},
Issuer: pkix.Name{CommonName: "issuer.test.example.com", Organization: []string{"Issuer"}},
SerialNumber: big.NewInt(serial),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
SignatureAlgorithm: x509.SHA256WithRSA,
NotBefore: time.Now().Add(-time.Hour).Truncate(time.Second).UTC(),
NotAfter: time.Now().Add(365 * 24 * time.Hour).Truncate(time.Second).UTC(),
BasicConstraintsValid: true,
}
return generateTestHostCertificateRecord(t, h.ID, tmpl)
}
ingest := func(t *testing.T, h *fleet.Host, recs ...*fleet.HostCertificateRecord) {
t.Helper()
require.NoError(t, ds.UpdateHostCertificates(ctx, h.ID, h.UUID, recs, fleet.HostCertificateOriginOsquery, nil))
}
ackVerified := func(t *testing.T, h *fleet.Host, cmdUUID string) {
t.Helper()
verified := fleet.MDMDeliveryVerified
require.NoError(t, updateMDMWindowsHostProfileStatusFromResponseDB(ctx, ds.writer(ctx),
[]*fleet.MDMWindowsProfilePayload{{HostUUID: h.UUID, CommandUUID: cmdUUID, Status: &verified}}))
}
// insertConfigProfile creates the team config profile row whose SyncML scope (./Device vs ./User) the backstop
// inspects when only the system store was observed.
insertConfigProfile := func(t *testing.T, profileUUID, name string, userScoped bool) {
t.Helper()
locURI := "./Device/Vendor/MSFT/ClientCertificateInstall/SCEP"
if userScoped {
locURI = "./User/Vendor/MSFT/ClientCertificateInstall/SCEP"
}
syncml := fmt.Sprintf(`<Replace><Item><Target><LocURI>%s/%s/Install/Enroll</LocURI></Target></Item></Replace>`, locURI, profileUUID)
_, err := ds.writer(ctx).ExecContext(ctx,
`INSERT INTO mdm_windows_configuration_profiles (profile_uuid, team_id, name, syncml, uploaded_at) VALUES (?, 0, ?, ?, NOW())`,
profileUUID, name, syncml)
require.NoError(t, err)
}
backdateProfile := func(t *testing.T, h *fleet.Host, profileUUID string, ago time.Duration) {
t.Helper()
_, err := ds.writer(ctx).ExecContext(ctx,
`UPDATE host_mdm_windows_profiles SET updated_at = DATE_SUB(NOW(), INTERVAL ? SECOND) WHERE host_uuid = ? AND profile_uuid = ?`,
int(ago.Seconds()), h.UUID, profileUUID)
require.NoError(t, err)
}
// plainCert builds an ingestable host cert that does NOT carry any profile's renewal-ID marker.
plainCert := func(t *testing.T, h *fleet.Host, cn string, source fleet.HostCertificateSource, username string) *fleet.HostCertificateRecord {
t.Helper()
sum := make([]byte, 20)
copy(sum, cn)
return &fleet.HostCertificateRecord{
HostID: h.ID,
CommonName: cn,
SubjectCommonName: cn,
SHA1Sum: sum,
NotValidBefore: time.Now().Add(-time.Hour),
NotValidAfter: time.Now().Add(365 * 24 * time.Hour),
Source: source,
Username: username,
}
}
// ACK status mapping: a proxied SCEP profile moves to "verifying" on the device's 2xx ACK
for i, tc := range []struct {
name string
caType fleet.CAConfigAssetType // "" means no managed-certificate row
want fleet.MDMDeliveryStatus
}{
{"proxied SCEP moves to verifying", fleet.CAConfigCustomSCEPProxy, fleet.MDMDeliveryVerifying},
{"non-certificate profile stays verified", "", fleet.MDMDeliveryVerified},
{"DigiCert profile stays verified", fleet.CAConfigDigiCert, fleet.MDMDeliveryVerified},
} {
t.Run("ACK: "+tc.name, func(t *testing.T) {
h := mkHost(t, fmt.Sprintf("ack-%d", i))
p, cmd := "w-ack", "cmd-ack"
upsertWinProfile(t, h, p, cmd, fleet.MDMDeliveryPending, 0)
if tc.caType != "" {
upsertHMMC(t, h, p, tc.caType, nil, nil)
}
ackVerified(t, h, cmd)
status, _, _ := getProfile(t, h, p)
require.Equal(t, tc.want, status)
})
}
t.Run("cert observation flips verifying to verified", func(t *testing.T) {
h := mkHost(t, "flip")
p := "w-scep-flip"
upsertWinProfile(t, h, p, "cmd-flip", fleet.MDMDeliveryVerifying, 0)
upsertHMMC(t, h, p, fleet.CAConfigCustomSCEPProxy, nil, nil)
ingest(t, h, certWithRenewalID(t, h, p, 7001))
status, _, _ := getProfile(t, h, p)
require.Equal(t, fleet.MDMDeliveryVerified, status)
})
t.Run("cert observation self-heals failed to verified and clears detail", func(t *testing.T) {
h := mkHost(t, "heal")
p := "w-scep-heal"
upsertWinProfile(t, h, p, "cmd-heal", fleet.MDMDeliveryFailed, 0)
upsertHMMC(t, h, p, fleet.CAConfigCustomSCEPProxy, nil, nil)
_, err := ds.writer(ctx).ExecContext(ctx,
`UPDATE host_mdm_windows_profiles SET detail = ? WHERE host_uuid = ? AND profile_uuid = ?`,
"SCEP PKIOperation failed: HTTP 500", h.UUID, p)
require.NoError(t, err)
ingest(t, h, certWithRenewalID(t, h, p, 7002))
status, detail, _ := getProfile(t, h, p)
require.Equal(t, fleet.MDMDeliveryVerified, status)
require.Empty(t, detail)
})
t.Run("no matching cert keeps profile verifying", func(t *testing.T) {
h := mkHost(t, "nomatch")
p := "w-scep-nomatch"
upsertWinProfile(t, h, p, "cmd-nomatch", fleet.MDMDeliveryVerifying, 0)
upsertHMMC(t, h, p, fleet.CAConfigCustomSCEPProxy, nil, nil)
// A cert for an unrelated profile: exercises the ingest path but matches nothing here.
ingest(t, h, certWithRenewalID(t, h, "w-unrelated-profile", 7003))
status, _, _ := getProfile(t, h, p)
require.Equal(t, fleet.MDMDeliveryVerifying, status)
})
t.Run("DigiCert profile not flipped even with observed cert dates", func(t *testing.T) {
h := mkHost(t, "digicert-flip")
p := "w-digicert-flip"
upsertWinProfile(t, h, p, "cmd-digicert-flip", fleet.MDMDeliveryVerifying, 0)
nvb := time.Now().Add(-time.Hour).Truncate(time.Second).UTC()
nva := time.Now().Add(365 * 24 * time.Hour).Truncate(time.Second).UTC()
upsertHMMC(t, h, p, fleet.CAConfigDigiCert, &nvb, &nva)
ingest(t, h, certWithRenewalID(t, h, "w-trigger-only", 7004))
status, _, _ := getProfile(t, h, p)
require.Equal(t, fleet.MDMDeliveryVerifying, status)
})
t.Run("SetMDMWindowsHostProfileFailed marks failed and preserves retries", func(t *testing.T) {
h := mkHost(t, "setfailed")
p := "w-fail"
upsertWinProfile(t, h, p, "cmd-fail", fleet.MDMDeliveryVerifying, 1)
require.NoError(t, ds.SetMDMWindowsHostProfileFailed(ctx, h.UUID, p, "SCEP PKIOperation failed: HTTP 500"))
status, detail, retries := getProfile(t, h, p)
require.Equal(t, fleet.MDMDeliveryFailed, status)
require.Equal(t, "SCEP PKIOperation failed: HTTP 500", detail)
require.Equal(t, 1, retries)
})
t.Run("SetMDMWindowsHostProfileFailed does not clobber verified", func(t *testing.T) {
h := mkHost(t, "setfailed-verified")
p := "w-fail-verified"
upsertWinProfile(t, h, p, "cmd-fail-verified", fleet.MDMDeliveryVerified, 0)
require.NoError(t, ds.SetMDMWindowsHostProfileFailed(ctx, h.UUID, p, "SCEP GetCACert failed: timeout"))
status, _, _ := getProfile(t, h, p)
require.Equal(t, fleet.MDMDeliveryVerified, status)
})
t.Run("SetMDMWindowsHostProfileFailed no-ops for a removed profile", func(t *testing.T) {
h := mkHost(t, "setfailed-missing")
// No profile row exists for this (host, profile): must not error and must not resurrect a row.
require.NoError(t, ds.SetMDMWindowsHostProfileFailed(ctx, h.UUID, "w-missing", "SCEP PKIOperation failed: HTTP 403"))
var count int
require.NoError(t, sqlx.GetContext(ctx, ds.reader(ctx), &count,
`SELECT COUNT(*) FROM host_mdm_windows_profiles WHERE host_uuid = ? AND profile_uuid = ?`, h.UUID, "w-missing"))
require.Equal(t, 0, count)
})
// Verification backstop: once the grace period elapses and a report proves the certificate's store was readable
// but the cert is absent, the profile fails. Device-scoped relies on the always-readable system store; user-scoped
// needs a user cert in the report (single-user assumption).
for i, tc := range []struct {
name string
userScoped bool
backdate time.Duration // 0 = still within the grace window
certSource fleet.HostCertificateSource
username string
retries int
wantStatus fleet.MDMDeliveryStatus
wantDetail string
}{
{"device-scoped fails past grace when cert absent", false, 2 * time.Hour, fleet.SystemHostCertificate, "", 2, fleet.MDMDeliveryFailed, windowsSCEPCertNotFoundDetail},
{"device-scoped stays verifying within grace", false, 0, fleet.SystemHostCertificate, "", 0, fleet.MDMDeliveryVerifying, ""},
{"user-scoped stays verifying when no user cert observed", true, 2 * time.Hour, fleet.SystemHostCertificate, "", 0, fleet.MDMDeliveryVerifying, ""},
{"user-scoped fails past grace once a user cert is observed", true, 2 * time.Hour, fleet.UserHostCertificate, "alice", 0, fleet.MDMDeliveryFailed, windowsSCEPCertNotFoundDetail},
} {
t.Run("backstop: "+tc.name, func(t *testing.T) {
h := mkHost(t, fmt.Sprintf("backstop-%d", i))
p := fmt.Sprintf("w-backstop-%d", i)
upsertWinProfile(t, h, p, "cmd-backstop", fleet.MDMDeliveryVerifying, tc.retries)
insertConfigProfile(t, p, fmt.Sprintf("backstop-%d", i), tc.userScoped)
upsertHMMC(t, h, p, fleet.CAConfigCustomSCEPProxy, nil, nil)
if tc.backdate > 0 {
backdateProfile(t, h, p, tc.backdate)
}
// The ingested cert never carries this profile's renewal-ID marker; only its source (system vs user)
// varies, which decides whether the user store is proven readable.
ingest(t, h, plainCert(t, h, "some-cert", tc.certSource, tc.username))
status, detail, retries := getProfile(t, h, p)
require.Equal(t, tc.wantStatus, status)
require.Equal(t, tc.wantDetail, detail)
require.Equal(t, tc.retries, retries)
})
}
t.Run("observed cert wins over the backstop even past grace", func(t *testing.T) {
h := mkHost(t, "backstop-flipwins")
p := "w-backstop-flipwins"
upsertWinProfile(t, h, p, "cmd-bs-flipwins", fleet.MDMDeliveryVerifying, 0)
insertConfigProfile(t, p, "bs-flipwins", false)
upsertHMMC(t, h, p, fleet.CAConfigCustomSCEPProxy, nil, nil)
backdateProfile(t, h, p, 2*time.Hour)
// The matching cert is present, so the verified-flip must win and the backstop must not fire.
ingest(t, h, certWithRenewalID(t, h, p, 7101))
status, _, _ := getProfile(t, h, p)
require.Equal(t, fleet.MDMDeliveryVerified, status)
})
}
func testUpdateAndListHostCertificates(t *testing.T, ds *Datastore) {
ctx := t.Context()
+75
View File
@@ -1166,6 +1166,36 @@ ON DUPLICATE KEY UPDATE
return result, nil
}
// renewalIDManagedCertProfileUUIDsDB returns, among the given profile UUIDs, those that have a managed-certificate row
// for the host whose CA type carries a renewal-ID marker (custom SCEP proxy, NDES, or Smallstep).
func renewalIDManagedCertProfileUUIDsDB(ctx context.Context, tx sqlx.ExtContext, hostUUID string, profileUUIDs []string) (map[string]struct{}, error) {
if len(profileUUIDs) == 0 {
return nil, nil
}
caTypes := fleet.ListCATypesWithRenewalIDSupport()
caTypeStrs := make([]string, 0, len(caTypes))
for _, t := range caTypes {
caTypeStrs = append(caTypeStrs, string(t))
}
stmt, args, err := sqlx.In(`
SELECT profile_uuid
FROM host_mdm_managed_certificates
WHERE host_uuid = ? AND profile_uuid IN (?) AND type IN (?)`,
hostUUID, profileUUIDs, caTypeStrs)
if err != nil {
return nil, ctxerr.Wrap(ctx, err, "building managed cert profile query")
}
var uuids []string
if err := sqlx.SelectContext(ctx, tx, &uuids, stmt, args...); err != nil {
return nil, ctxerr.Wrap(ctx, err, "selecting renewal-ID managed cert profile uuids")
}
result := make(map[string]struct{}, len(uuids))
for _, u := range uuids {
result[u] = struct{}{}
}
return result, nil
}
// updateMDMWindowsHostProfileStatusFromResponseDB takes a slice of potential
// profile payloads and updates the corresponding `status` and `detail` columns
// in `host_mdm_windows_profiles`
@@ -1226,12 +1256,41 @@ func updateMDMWindowsHostProfileStatusFromResponseDB(
return ctxerr.Wrap(ctx, err, "running query to get matching profiles")
}
// Proxied SCEP profiles must not report "verified" off the device's SyncML ACK alone: a 2xx ACK only means the
// SCEP <Exec> was accepted by the CSP, not that a certificate was issued (the exchange runs asynchronously after).
// Downgrade those installs to "verifying" and let UpdateHostCertificates flip them to "verified" once the matching
// certificate is observed on the host. Detect them by an existing renewal-ID-backed managed-certificate row (custom
// SCEP proxy, NDES, or Smallstep).
var verifiedInstallProfileUUIDs []string
for _, hp := range matchingHostProfiles {
payload := uuidsToPayloads[hp.CommandUUID]
if payload == nil {
continue
}
if hp.OperationType == fleet.MDMOperationTypeInstall && payload.Status != nil && *payload.Status == fleet.MDMDeliveryVerified {
verifiedInstallProfileUUIDs = append(verifiedInstallProfileUUIDs, hp.ProfileUUID)
}
}
scepProxyProfileUUIDs, err := renewalIDManagedCertProfileUUIDsDB(ctx, tx, hostUUID, verifiedInstallProfileUUIDs)
if err != nil {
return ctxerr.Wrap(ctx, err, "checking for proxied SCEP managed certificate profiles")
}
// Partition matching entries into upsert and delete buckets.
var sb strings.Builder
args = args[:0]
var deleteCommandUUIDs []string
for _, hp := range matchingHostProfiles {
payload := uuidsToPayloads[hp.CommandUUID]
if payload == nil {
continue
}
if hp.OperationType == fleet.MDMOperationTypeInstall && payload.Status != nil && *payload.Status == fleet.MDMDeliveryVerified {
if _, ok := scepProxyProfileUUIDs[hp.ProfileUUID]; ok {
verifying := fleet.MDMDeliveryVerifying
payload.Status = &verifying
}
}
if payload.Status != nil && *payload.Status == fleet.MDMDeliveryFailed {
// Don't retry remove operations; removal is best-effort. Only retry install operations up to the max retry count.
if hp.OperationType != fleet.MDMOperationTypeRemove && hp.Retries < mdm.MaxWindowsProfileRetries {
@@ -1285,6 +1344,22 @@ func updateMDMWindowsHostProfileStatusFromResponseDB(
return nil
}
func (ds *Datastore) SetMDMWindowsHostProfileFailed(ctx context.Context, hostUUID string, profileUUID string, detail string) error {
// Only touch an existing install row (a removed profile is not resurrected). Never overwrite a row that already
// reached "verified" (the certificate was observed, so a late/stale upstream error must not regress it).
const stmt = `
UPDATE host_mdm_windows_profiles
SET status = ?, detail = ?
WHERE host_uuid = ? AND profile_uuid = ? AND operation_type = ?
AND (status IS NULL OR status <> ?)`
if _, err := ds.writer(ctx).ExecContext(ctx, stmt,
fleet.MDMDeliveryFailed, detail, hostUUID, profileUUID, fleet.MDMOperationTypeInstall, fleet.MDMDeliveryVerified,
); err != nil {
return ctxerr.Wrap(ctx, err, "set windows host profile failed")
}
return nil
}
func (ds *Datastore) GetMDMWindowsCommandResults(ctx context.Context, commandUUID string, hostUUID string) ([]*fleet.MDMCommandResult, error) {
query := `SELECT
mwe.host_uuid,
+4
View File
@@ -2292,6 +2292,10 @@ type Datastore interface {
// to be resent upon the next cron run.
ResendHostMDMProfile(ctx context.Context, hostUUID string, profileUUID string) error
// SetMDMWindowsHostProfileFailed marks the install row for the given (hostUUID, profileUUID) Windows profile as
// "failed" with the provided detail.
SetMDMWindowsHostProfileFailed(ctx context.Context, hostUUID string, profileUUID string, detail string) error
// BatchResendMDMProfileToHosts updates the profile status to NULL for the
// matching hosts that satisfy the filter, thereby triggering the profile to
// be resent upon the next cron run.
+12
View File
@@ -1408,6 +1408,8 @@ type ListMDMConfigProfilesFunc func(ctx context.Context, teamID *uint, opt fleet
type ResendHostMDMProfileFunc func(ctx context.Context, hostUUID string, profileUUID string) error
type SetMDMWindowsHostProfileFailedFunc func(ctx context.Context, hostUUID string, profileUUID string, detail string) error
type BatchResendMDMProfileToHostsFunc func(ctx context.Context, profileUUID string, filters fleet.BatchResendMDMProfileFilters) (int64, error)
type GetMDMConfigProfileStatusFunc func(ctx context.Context, profileUUID string) (fleet.MDMConfigProfileStatus, error)
@@ -4223,6 +4225,9 @@ type DataStore struct {
ResendHostMDMProfileFunc ResendHostMDMProfileFunc
ResendHostMDMProfileFuncInvoked bool
SetMDMWindowsHostProfileFailedFunc SetMDMWindowsHostProfileFailedFunc
SetMDMWindowsHostProfileFailedFuncInvoked bool
BatchResendMDMProfileToHostsFunc BatchResendMDMProfileToHostsFunc
BatchResendMDMProfileToHostsFuncInvoked bool
@@ -10177,6 +10182,13 @@ func (s *DataStore) ResendHostMDMProfile(ctx context.Context, hostUUID string, p
return s.ResendHostMDMProfileFunc(ctx, hostUUID, profileUUID)
}
func (s *DataStore) SetMDMWindowsHostProfileFailed(ctx context.Context, hostUUID string, profileUUID string, detail string) error {
s.mu.Lock()
s.SetMDMWindowsHostProfileFailedFuncInvoked = true
s.mu.Unlock()
return s.SetMDMWindowsHostProfileFailedFunc(ctx, hostUUID, profileUUID, detail)
}
func (s *DataStore) BatchResendMDMProfileToHosts(ctx context.Context, profileUUID string, filters fleet.BatchResendMDMProfileFilters) (int64, error) {
s.mu.Lock()
s.BatchResendMDMProfileToHostsFuncInvoked = true
@@ -8481,7 +8481,9 @@ func testWindowsSCEPProfile(s *integrationMDMTestSuite, windowsScepProfile []byt
scepCount := verifyCommands(1, syncml.CmdStatusOK)
require.Equal(t, 1, scepCount, "SCEP exchange should have run exactly once")
// Verify profile status is Verified due to successful response
// The device ACKed the SCEP <Exec>, but for a Fleet-proxied SCEP profile that only means the exchange was
// accepted, not that a certificate landed on the host. The profile stays "verifying" until Fleet observes the
// matching certificate.
profiles, err = s.ds.GetHostMDMWindowsProfiles(ctx, host.UUID)
require.NoError(t, err)
foundProfile = false
@@ -8491,6 +8493,34 @@ func testWindowsSCEPProfile(s *integrationMDMTestSuite, windowsScepProfile []byt
foundProfile = true
profileUUID = p.ProfileUUID
require.NotNil(t, p.Status)
assert.Equal(t, fleet.MDMDeliveryVerifying, *p.Status)
}
}
require.True(t, foundProfile, "WindowsSCEPProfile not found for host")
// Simulate osquery reporting the issued certificate. It carries the profile's renewal-ID marker
// (fleet-<profile_uuid>) in its OU, which Fleet matches back to the profile's managed-certificate row.
sha1Sum := make([]byte, 20)
copy(sha1Sum, profileUUID)
require.NoError(t, s.ds.UpdateHostCertificates(ctx, host.ID, host.UUID, []*fleet.HostCertificateRecord{{
HostID: host.ID,
CommonName: "windows-scep-cert",
SubjectCommonName: "windows-scep-cert",
SubjectOrganizationalUnit: "fleet-" + profileUUID,
SHA1Sum: sha1Sum,
NotValidBefore: time.Now().Add(-time.Hour),
NotValidAfter: time.Now().Add(365 * 24 * time.Hour),
Source: fleet.SystemHostCertificate,
}}, fleet.HostCertificateOriginOsquery, nil))
// Now that Fleet has observed the matching certificate, the profile is verified.
profiles, err = s.ds.GetHostMDMWindowsProfiles(ctx, host.UUID)
require.NoError(t, err)
foundProfile = false
for _, p := range profiles {
if p.Name == "WindowsSCEPProfile" {
foundProfile = true
require.NotNil(t, p.Status)
assert.EqualValues(t, fleet.MDMDeliveryVerified, *p.Status)
}
}