Files
Victor LyuboslavskyandLucas Manuel Rodriguez 8a28b83b00 Redis host cache optimizations (#46458)
<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #46338 

Changes load tested (osquery load test).
<img width="1532" height="282" alt="image"
src="https://github.com/user-attachments/assets/9eb38629-918e-4e05-bc0b-e2ae22e8b148"
/>
<img width="1414" height="937" alt="image"
src="https://github.com/user-attachments/assets/1b90afd3-087c-452e-939f-b495df3e59c8"
/>


# Checklist for submitter

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.

## Testing

- [x] Added/updated automated tests
- [x] QA'd all new/changed functionality manually


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **Performance Improvements**
* Extended host cache retention period from 60 seconds to 180 seconds to
improve overall cache efficiency and hit rates.
* Optimized host cache invalidation strategy to eliminate unnecessary
reverse-index lookups, significantly reducing database reader load and
Redis CPU consumption.

* **Tests**
* Added test coverage for host update cache invalidation edge cases to
ensure proper behavior.

<!-- review_stack_entry_start -->

[![Review Change
Stack](https://storage.googleapis.com/coderabbit_public_assets/review-stack-in-coderabbit-ui.svg)](https://app.coderabbit.ai/change-stack/fleetdm/fleet/pull/46458?utm_source=github_walkthrough&utm_medium=github&utm_campaign=change_stack)

<!-- review_stack_entry_end -->

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Lucas Manuel Rodriguez <lucas@fleetdm.com>
2026-06-01 08:12:50 -05:00

211 lines
9.1 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package mysqlredis
import (
"context"
"time"
"github.com/fleetdm/fleet/v4/server/fleet"
)
// This file holds the mysqlredis overrides for write-path methods that mutate
// cached host fields. Each wrapper delegates to the inner Datastore first, then
// invalidates the Redis-backed host cache on success. Errors from the inner
// call short-circuit invalidation. We must not poison the cache on transient
// failures.
//
// Invalidation reason labels must be one of the low-cardinality values the
// metric in metrics.go expects: update | enroll | team | delete | cert.
//
// NOTE: If hostCacheEnabled is false, every helper used here is a no-op and
// these wrappers behave identically to the inner methods.
// UpdateHost wraps the inner UpdateHost to invalidate the cache entry for the
// host's current node_key after a successful write. The old node_key entry (if
// re-enrollment changed it) becomes unreachable and expires via TTL.
func (d *Datastore) UpdateHost(ctx context.Context, host *fleet.Host) error {
if err := d.Datastore.UpdateHost(ctx, host); err != nil {
return err
}
d.invalidateAfterHostUpdate(ctx, host, "update")
return nil
}
// SerialUpdateHost wraps the inner SerialUpdateHost. SerialUpdateHost enqueues
// to the datastore's writeCh and blocks until the write completes, so on
// return the UPDATE has already hit MySQL and we can safely invalidate. This
// wrap is essential because the writeCh loop in mysql.go invokes the inner
// UpdateHost directly and bypasses any wrapper on that method.
func (d *Datastore) SerialUpdateHost(ctx context.Context, host *fleet.Host) error {
if err := d.Datastore.SerialUpdateHost(ctx, host); err != nil {
return err
}
d.invalidateAfterHostUpdate(ctx, host, "update")
return nil
}
// UpdateHostOsqueryIntervals invalidates by host ID after the inner write.
// Affected fields: distributed_interval, config_tls_refresh, logger_tls_period.
func (d *Datastore) UpdateHostOsqueryIntervals(ctx context.Context, hostID uint, intervals fleet.HostOsqueryIntervals) error {
if err := d.Datastore.UpdateHostOsqueryIntervals(ctx, hostID, intervals); err != nil {
return err
}
d.hostCacheDeleteByID(ctx, hostID, "update")
return nil
}
// UpdateHostRefetchRequested invalidates by host ID after the inner write.
// Affected field: refetch_requested. Staleness would cause admin-triggered
// refetches to be delayed by up to TTL; invalidation keeps that latency low.
func (d *Datastore) UpdateHostRefetchRequested(ctx context.Context, hostID uint, value bool) error {
if err := d.Datastore.UpdateHostRefetchRequested(ctx, hostID, value); err != nil {
return err
}
d.hostCacheDeleteByID(ctx, hostID, "update")
return nil
}
// UpdateHostRefetchCriticalQueriesUntil invalidates by host ID after the inner
// write. Affected field: refetch_critical_queries_until.
func (d *Datastore) UpdateHostRefetchCriticalQueriesUntil(ctx context.Context, hostID uint, until *time.Time) error {
if err := d.Datastore.UpdateHostRefetchCriticalQueriesUntil(ctx, hostID, until); err != nil {
return err
}
d.hostCacheDeleteByID(ctx, hostID, "update")
return nil
}
// EnrollOrbit invalidates for the returned host on successful enrollment. Orbit enrollment may create a new
// hosts row or update an existing one's orbit_node_key + team_id; in either case the cached snapshot is stale.
//
// Asymmetry with EnrollOsquery: mysql.EnrollOsquery does a final SELECT that hydrates host.NodeKey and
// host.OrbitNodeKey on the returned struct, but mysql.EnrollOrbit does NOT, the *fleet.Host it returns has
// only ID/ComputerName/Hostname/HardwareModel/HardwareSerial/Platform/PlatformLike populated.
//
// Replay opts to recover the just-issued orbit_node_key so the negative-cache plug fires for orbit, matching
// the contract the helper is documented to provide. Without this, an /orbit/* probe arriving in the 0-5s
// window before EnrollOrbit committed leaves an onk_miss:<K> entry that survives the wrapper, and the
// freshly-enrolled host returns NotFound from the negative cache for up to hostCacheNegativeTTL on its
// first /orbit/* requests.
func (d *Datastore) EnrollOrbit(ctx context.Context, opts ...fleet.DatastoreEnrollOrbitOption) (*fleet.Host, error) {
host, err := d.Datastore.EnrollOrbit(ctx, opts...)
if err != nil {
return nil, err
}
cfg := &fleet.DatastoreEnrollOrbitConfig{}
for _, opt := range opts {
opt(cfg)
}
if host != nil && host.ID != 0 {
d.hostCacheDeleteByID(ctx, host.ID, "enroll")
d.hostCacheClearDirectEntries(ctx, "", cfg.OrbitNodeKey)
}
return host, nil
}
// AddHostsToTeam invalidates every host in the batch after a successful team
// reassignment. Uses the pipelined batch invalidator (one MGET + one DEL per
// Redis slot, chunked) rather than calling hostCacheDeleteByID in a loop —
// that naive approach takes ~8 sequential Redis round-trips per host and at
// 10k hosts × ~1 ms RTT adds ~80 s to the API call. The batched version is
// O(slots × chunks) and stays synchronous on return.
func (d *Datastore) AddHostsToTeam(ctx context.Context, params *fleet.AddHostsToTeamParams) error {
if err := d.Datastore.AddHostsToTeam(ctx, params); err != nil {
return err
}
if params != nil {
d.invalidateHostIDs(ctx, params.HostIDs, "team")
}
return nil
}
// UpdateHostIdentityCertHostIDBySerial invalidates for the host whose
// has_host_identity_cert just flipped. This is the security-critical path: a
// stale `false` would cause AuthenticateHost to skip the httpsig verification
// for up to TTL. Explicit invalidation closes that window to a round-trip.
func (d *Datastore) UpdateHostIdentityCertHostIDBySerial(ctx context.Context, serialNumber uint64, hostID uint) error {
if err := d.Datastore.UpdateHostIdentityCertHostIDBySerial(ctx, serialNumber, hostID); err != nil {
return err
}
d.hostCacheDeleteByID(ctx, hostID, "cert")
return nil
}
// invalidateAfterHostUpdate is the common tail for write paths that update an already-existing host
// (UpdateHost, SerialUpdateHost). DELs the known node keys directly via pipelinedDEL, skipping the two
// id2nk/id2onk reverse-index GETs that hostCacheDeleteByID would perform.
//
// Fallback rule is asymmetric and follows real host shapes in the hosts table:
// - NodeKey absent: anomalous. Every osquery-capable platform (darwin/windows/linux/android) has a
// non-null node_key, and hosts without a node_key never enter this cache. A caller arriving here with no NodeKey
// is loading a sparsely-populated host, so fall back to hostCacheDeleteByID and let the reverse
// indices resolve whatever is cached.
// - OrbitNodeKey absent: normal. Run osquery without Orbit. no LoadHostByOrbitNodeKey ever populated
//
// This path does NOT clear by direct keys to cover the pre-enrollment-negative-cache race that motivates
// invalidateAfterHostEnroll's direct-keys clear: that race cannot apply to UpdateHost callers (you cannot
// UPDATE a host that doesn't exist yet), so no negative entry from a probe-before-row can survive here.
func (d *Datastore) invalidateAfterHostUpdate(ctx context.Context, host *fleet.Host, reason string) {
if host == nil || host.ID == 0 || !d.hostCacheEnabled {
return
}
nk := ""
if host.NodeKey != nil {
nk = *host.NodeKey
}
onk := ""
if host.OrbitNodeKey != nil {
onk = *host.OrbitNodeKey
}
if nk == "" {
d.hostCacheDeleteByID(ctx, host.ID, reason)
return
}
keys := []string{
hostCacheKeyByNodeKey(nk),
hostCacheKeyMiss(nk),
hostCacheIndexByID(host.ID),
}
if onk != "" {
keys = append(keys, hostCacheKeyByOrbitNodeKey(onk), hostCacheKeyOrbitMiss(onk))
}
// Always include id2onk in the DEL list. If the host has an orbit_node_key the index needs to be
// cleared along with the rest; if it doesn't, the DEL is a no-op (DEL on a missing key returns 0).
keys = append(keys, hostCacheOrbitIndexByID(host.ID))
d.pipelinedDEL(ctx, keys)
d.recordHostCacheInvalidation(ctx, reason)
}
// invalidateAfterHostEnroll is the common tail for write paths that may CREATE a host or rotate its
// node_key (NewHost, EnrollOsquery, EnrollOrbit). Does the by-ID invalidation that
// invalidateAfterHostUpdate does, plus a direct-keys clear using the caller-supplied NodeKey and
// OrbitNodeKey on the returned *fleet.Host.
//
// The direct-keys clear plugs the pre-enrollment-negative-cache race: if a poll arrived with the
// new node_key BEFORE the host row existed, LoadHostByNodeKey returned NotFound and wrote
// nk_miss:<new_key> with a 5s TTL but did NOT populate the reverse index (no host ID to point at).
// hostCacheDeleteByID's reverse-index walk cannot find that entry; only a direct DEL using the
// just-issued node_key can. Without the direct clear, the freshly-enrolled host's first 0-5s of
// auth attempts return NotFound from the negative cache.
//
// Does NOT record a second invalidation; hostCacheDeleteByID already bumped the counter.
func (d *Datastore) invalidateAfterHostEnroll(ctx context.Context, host *fleet.Host, reason string) {
if host == nil || host.ID == 0 {
return
}
d.hostCacheDeleteByID(ctx, host.ID, reason)
nk := ""
if host.NodeKey != nil {
nk = *host.NodeKey
}
onk := ""
if host.OrbitNodeKey != nil {
onk = *host.OrbitNodeKey
}
d.hostCacheClearDirectEntries(ctx, nk, onk)
}