Fix: certificate field too long error due to osquery not encoding non-ascii characters (#40095)
<!-- Add the related story/sub-task/bug number, like Resolves #123, or remove if NA --> **Related issue:** Resolves #37306 I think we should actually solve this at the osquery level, but this is just an alternative solution. # Checklist for submitter - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files) for more information. ## Testing - [x] Added/updated automated tests - [x] QA'd all new/changed functionality manually #### Setup On a macOS VM: 1. Generated a certificate with Cyrillic characters in CN and O fields: ```bash openssl req -x509 -newkey rsa:2048 -keyout newkey.pem -out newcert.pem -days 365 -nodes -utf8 -subj "/CN=АБВГДЕЖЗИКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯа/O=АБВГДЕЖЗИКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯа" ``` 2. Installed the certificate: ```bash sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain newcert.pem ``` #### Before Refetched vitals and observed certificate field too long error logs: ``` ts=2026-02-18T21:50:47Z level=error msg="truncating certificate field" err="certificate field too long" field=subject_org host_id=32 original_length=304 max_length=255 truncated_value=\xC3\x90\xC2\xA2\xC3\x90\xC2\xB5\xC3\x91\xC2\x81\xC3\x91\xC2\x82\xC3\x90\xC2\xBE\xC3\x90\xC2\xB2\xC3\x90\xC2\xB0\xC3\x91\xC2\x8F\xC3\x90\xC2\x9E\xC3\x91\xC2\x80\xC3\x90\xC2\xB3\xC3\x90\xC2\xB0\xC3\x90\xC2\xBD\xC3\x90\xC2\xB8\xC3\x90\xC2\xB7\xC3\x90\xC2\xB ts=2026-02-18T21:50:47Z level=error msg="truncating certificate field" err="certificate field too long" field=subject_common_name host_id=32 original_length=304 max_length=255 truncated_value=\xC3\x90\xC2\xA2\xC3\x90\xC2\xB5\xC3\x91\xC2\x81\xC3\x91\xC2\x82\xC3\x90\xC2\xBE\xC3\x90\xC2\xB2\xC3\x90\xC2\xB0\xC3\x91\xC2\x8F\xC3\x90\xC2\x9E\xC3\x91\xC2\x80\xC3\x90\xC2\xB3\xC3\x90\xC2\xB0\xC3\x90\xC2\xBD\xC3\x90\xC2\xB8\xC3\x90\xC2\xB7\xC3\x90\xC2\xB ts=2026-02-18T21:50:47Z level=error msg="truncating certificate field" err="certificate field too long" field=issuer_org host_id=32 original_length=304 max_length=255 truncated_value=\xC3\x90\xC2\xA2\xC3\x90\xC2\xB5\xC3\x91\xC2\x81\xC3\x91\xC2\x82\xC3\x90\xC2\xBE\xC3\x90\xC2\xB2\xC3\x90\xC2\xB0\xC3\x91\xC2\x8F\xC3\x90\xC2\x9E\xC3\x91\xC2\x80\xC3\x90\xC2\xB3\xC3\x90\xC2\xB0\xC3\x90\xC2\xBD\xC3\x90\xC2\xB8\xC3\x90\xC2\xB7\xC3\x90\xC2\xB ts=2026-02-18T21:50:47Z level=error msg="truncating certificate field" err="certificate field too long" field=issuer_common_name host_id=32 original_length=304 max_length=255 truncated_value=\xC3\x90\xC2\xA2\xC3\x90\xC2\xB5\xC3\x91\xC2\x81\xC3\x91\xC2\x82\xC3\x90\xC2\xBE\xC3\x90\xC2\xB2\xC3\x90\xC2\xB0\xC3\x91\xC2\x8F\xC3\x90\xC2\x9E\xC3\x91\xC2\x80\xC3\x90\xC2\xB3\xC3\x90\xC2\xB0\xC3\x90\xC2\xBD\xC3\x90\xC2\xB8\xC3\x90\xC2\xB7\xC3\x90\xC2\xB ts=2026-02-18T21:50:47Z level=error msg="truncating certificate field" err="certificate field too long" field=subject_org host_id=32 original_length=256 max_length=255 truncated_value=\xD0\x90\xD0\x91\xD0\x92\xD0\x93\xD0\x94\xD0\x95\xD0\x96\xD0\x97\xD0\x98\xD0\x9A\xD0\x9B\xD0\x9C\xD0\x9D\xD0\x9E\xD0\x9F\xD0\xA0\xD0\xA1\xD0\xA2\xD0\xA3\xD0\xA4\xD0\xA5\xD0\xA6\xD0\xA7\xD0\xA8\xD0\xA9\xD0\xAA\xD0\xAB\xD0\xAC\xD0\xAD\xD0\xAE\xD0\xAF\xD0\xB ts=2026-02-18T21:50:47Z level=error msg="truncating certificate field" err="certificate field too long" field=subject_common_name host_id=32 original_length=256 max_length=255 truncated_value=\xD0\x90\xD0\x91\xD0\x92\xD0\x93\xD0\x94\xD0\x95\xD0\x96\xD0\x97\xD0\x98\xD0\x9A\xD0\x9B\xD0\x9C\xD0\x9D\xD0\x9E\xD0\x9F\xD0\xA0\xD0\xA1\xD0\xA2\xD0\xA3\xD0\xA4\xD0\xA5\xD0\xA6\xD0\xA7\xD0\xA8\xD0\xA9\xD0\xAA\xD0\xAB\xD0\xAC\xD0\xAD\xD0\xAE\xD0\xAF\xD0\xB ts=2026-02-18T21:50:47Z level=error msg="truncating certificate field" err="certificate field too long" field=issuer_org host_id=32 original_length=256 max_length=255 truncated_value=\xD0\x90\xD0\x91\xD0\x92\xD0\x93\xD0\x94\xD0\x95\xD0\x96\xD0\x97\xD0\x98\xD0\x9A\xD0\x9B\xD0\x9C\xD0\x9D\xD0\x9E\xD0\x9F\xD0\xA0\xD0\xA1\xD0\xA2\xD0\xA3\xD0\xA4\xD0\xA5\xD0\xA6\xD0\xA7\xD0\xA8\xD0\xA9\xD0\xAA\xD0\xAB\xD0\xAC\xD0\xAD\xD0\xAE\xD0\xAF\xD0\xB ts=2026-02-18T21:50:47Z level=error msg="truncating certificate field" err="certificate field too long" field=issuer_common_name host_id=32 original_length=256 max_length=255 truncated_value=\xD0\x90\xD0\x91\xD0\x92\xD0\x93\xD0\x94\xD0\x95\xD0\x96\xD0\x97\xD0\x98\xD0\x9A\xD0\x9B\xD0\x9C\xD0\x9D\xD0\x9E\xD0\x9F\xD0\xA0\xD0\xA1\xD0\xA2\xD0\xA3\xD0\xA4\xD0\xA5\xD0\xA6\xD0\xA7\xD0\xA8\xD0\xA9\xD0\xAA\xD0\xAB\xD0\xAC\xD0\xAD\xD0\xAE\xD0\xAF\xD0\xB ``` https://github.com/user-attachments/assets/31b80643-34fb-42ee-b3a1-19d54fe1d516 #### After https://github.com/user-attachments/assets/d49c1643-0920-4607-a2f1-fa6bb658c16b
This commit is contained in:
Nico
@@ -0,0 +1 @@
|
||||
* Fixed truncation of certificate fields containing non-ASCII characters.
|
||||
@@ -3,6 +3,7 @@ package fleet
|
||||
import (
|
||||
"crypto/sha1" // nolint:gosec // used for compatibility with existing osquery certificates table schema
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -302,6 +303,33 @@ func parseWindowsDN(dn string) (*HostCertificateNameDetails, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DecodeHexEscapes replaces literal \xHH escape sequences with the actual byte values.
|
||||
// For example, the string `\xD0\x90` (8 ASCII characters) becomes the 2-byte UTF-8 sequence for the Cyrillic letter "А".
|
||||
// Returns the original string unchanged if no escape sequences are found.
|
||||
// Incomplete or invalid sequences (e.g. `\xZZ`, `\x` at end of string) are left as-is.
|
||||
func DecodeHexEscapes(s string) string {
|
||||
if !strings.Contains(s, `\x`) {
|
||||
return s
|
||||
}
|
||||
|
||||
var buf strings.Builder
|
||||
buf.Grow(len(s))
|
||||
i := 0
|
||||
for i < len(s) {
|
||||
if i+3 < len(s) && s[i] == '\\' && s[i+1] == 'x' {
|
||||
b, err := hex.DecodeString(s[i+2 : i+4])
|
||||
if err == nil {
|
||||
buf.Write(b)
|
||||
i += 4
|
||||
continue
|
||||
}
|
||||
}
|
||||
buf.WriteByte(s[i])
|
||||
i++
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func firstOrEmpty(s []string) string {
|
||||
if len(s) > 0 {
|
||||
return s[0]
|
||||
|
||||
@@ -229,3 +229,69 @@ func TestExtractHostCertificateFromMDMAppleCertificateList(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeHexEscapes(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "plain ASCII unchanged",
|
||||
input: "hello world",
|
||||
expected: "hello world",
|
||||
},
|
||||
{
|
||||
name: "empty string",
|
||||
input: "",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "cyrillic hex escapes",
|
||||
input: `\xD0\x90\xD0\x91`,
|
||||
expected: "АБ",
|
||||
},
|
||||
{
|
||||
name: "mixed ASCII and hex escapes",
|
||||
input: `abc\xD0\x90def`,
|
||||
expected: "abcАdef",
|
||||
},
|
||||
{
|
||||
name: "full DN string with hex escapes",
|
||||
input: `/O=\xD0\x90\xD0\x91/CN=\xD0\x92\xD0\x93`,
|
||||
expected: "/O=АБ/CN=ВГ",
|
||||
},
|
||||
{
|
||||
name: "incomplete escape at end of string",
|
||||
input: `hello\x`,
|
||||
expected: `hello\x`,
|
||||
},
|
||||
{
|
||||
name: "incomplete escape with one hex char at end",
|
||||
input: `hello\xD`,
|
||||
expected: `hello\xD`,
|
||||
},
|
||||
{
|
||||
name: "invalid hex after backslash-x",
|
||||
input: `hello\xZZworld`,
|
||||
expected: `hello\xZZworld`,
|
||||
},
|
||||
{
|
||||
name: "no escape sequences present",
|
||||
input: `/C=US/O=Fleet Device Management Inc./CN=FleetDM`,
|
||||
expected: `/C=US/O=Fleet Device Management Inc./CN=FleetDM`,
|
||||
},
|
||||
{
|
||||
name: "backslash not followed by x",
|
||||
input: `hello\nworld`,
|
||||
expected: `hello\nworld`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result := DecodeHexEscapes(tc.input)
|
||||
assert.Equal(t, tc.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3292,6 +3292,12 @@ func directIngestHostCertificatesDarwin(
|
||||
|
||||
certs := make([]*fleet.HostCertificateRecord, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
// Unescape \xHH sequences in fields that may contain non-ASCII
|
||||
// characters (e.g. Cyrillic) in the certificate's distinguished name.
|
||||
row["common_name"] = fleet.DecodeHexEscapes(row["common_name"])
|
||||
row["subject"] = fleet.DecodeHexEscapes(row["subject"])
|
||||
row["issuer"] = fleet.DecodeHexEscapes(row["issuer"])
|
||||
|
||||
csum, err := hex.DecodeString(row["sha1"])
|
||||
if err != nil {
|
||||
logger.ErrorContext(ctx, "decoding sha1", "component", "service", "method", "directIngestHostCertificates", "err", err)
|
||||
@@ -3379,6 +3385,12 @@ func directIngestHostCertificatesWindows(
|
||||
// SHA1 sum + username
|
||||
existsSha1User := make(map[string]bool, len(rows))
|
||||
for _, row := range rows {
|
||||
// Unescape \xHH sequences in fields that may contain non-ASCII
|
||||
// characters (e.g. Cyrillic) in the certificate's distinguished name.
|
||||
row["common_name"] = fleet.DecodeHexEscapes(row["common_name"])
|
||||
row["subject"] = fleet.DecodeHexEscapes(row["subject"])
|
||||
row["issuer"] = fleet.DecodeHexEscapes(row["issuer"])
|
||||
|
||||
csum, err := hex.DecodeString(row["sha1"])
|
||||
if err != nil {
|
||||
logger.ErrorContext(ctx, "decoding sha1", "component", "service", "method", "directIngestHostCertificates", "err", err)
|
||||
|
||||
@@ -2486,6 +2486,52 @@ func TestDirectIngestHostCertificates(t *testing.T) {
|
||||
require.True(t, ds.UpdateHostCertificatesFuncInvoked)
|
||||
}
|
||||
|
||||
func TestDirectIngestHostCertificatesDarwinHexEscapes(t *testing.T) {
|
||||
ds := new(mock.Store)
|
||||
ctx := t.Context()
|
||||
logger := slog.New(slog.DiscardHandler)
|
||||
host := &fleet.Host{ID: 1, UUID: "host-uuid", Platform: "darwin"}
|
||||
|
||||
// Simulate osquery outputting Cyrillic characters as literal \xHH escape
|
||||
// sequences. "АБ" in UTF-8 is bytes D0 90 D0 91, which osquery returns as
|
||||
// the 16-character ASCII string `\xD0\x90\xD0\x91`.
|
||||
row := map[string]string{
|
||||
"ca": "0",
|
||||
"common_name": `\xD0\x90\xD0\x91`,
|
||||
"subject": `/C=US/O=\xD0\x90\xD0\x91/OU=\xD0\x92\xD0\x93/CN=\xD0\x94\xD0\x95`,
|
||||
"issuer": `/O=\xD0\x96\xD0\x97/CN=\xD0\x98\xD0\x9A`,
|
||||
"key_algorithm": "rsaEncryption",
|
||||
"key_strength": "2048",
|
||||
"key_usage": "Digital Signature",
|
||||
"serial": "abc123",
|
||||
"signing_algorithm": "sha256WithRSAEncryption",
|
||||
"not_valid_after": "1822755797",
|
||||
"not_valid_before": "1770228826",
|
||||
"sha1": "aabbccdd00112233445566778899aabbccddeeff",
|
||||
"source": "system",
|
||||
"path": "/Library/Keychains/System.keychain",
|
||||
}
|
||||
|
||||
ds.UpdateHostCertificatesFunc = func(ctx context.Context, hostID uint, hostUUID string, certs []*fleet.HostCertificateRecord) error {
|
||||
require.Len(t, certs, 1)
|
||||
cert := certs[0]
|
||||
|
||||
assert.Equal(t, "АБ", cert.CommonName)
|
||||
assert.Equal(t, "ДЕ", cert.SubjectCommonName)
|
||||
assert.Equal(t, "АБ", cert.SubjectOrganization)
|
||||
assert.Equal(t, "ВГ", cert.SubjectOrganizationalUnit)
|
||||
assert.Equal(t, "US", cert.SubjectCountry)
|
||||
assert.Equal(t, "ИК", cert.IssuerCommonName)
|
||||
assert.Equal(t, "ЖЗ", cert.IssuerOrganization)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
err := directIngestHostCertificatesDarwin(ctx, logger, host, ds, []map[string]string{row})
|
||||
require.NoError(t, err)
|
||||
require.True(t, ds.UpdateHostCertificatesFuncInvoked)
|
||||
}
|
||||
|
||||
func TestDirectIngestHostCertificatesWindows(t *testing.T) {
|
||||
ds := new(mock.Store)
|
||||
ctx := t.Context()
|
||||
|
||||
Reference in New Issue
Block a user