Add server-side orbit debug logging enablement - currently only configurable as a duration-after-enrollment setting (#45367)

<!-- Add the related story/sub-task/bug number, like Resolves #123, or
remove if NA -->
**Related issue:** Resolves #43997 

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/guides/committing-changes.md#changes-files)
for more information.

- [x] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements), JS
inline code is prevented especially for url redirects, and untrusted
data interpolated into shell scripts/commands is validated against shell
metacharacters.
- [x] Timeouts are implemented and retries are limited to avoid infinite
loops
- [x] If paths of existing endpoints are modified without backwards
compatibility, checked the frontend/CLI for any necessary changes

## Testing

- [x] Added/updated automated tests
- [x] Where appropriate, [automated tests simulate multiple hosts and
test for host
isolation](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/reference/patterns-backend.md#unit-testing)
(updates to one hosts's records do not affect another)

- [x] QA'd all new/changed functionality manually

## Database migrations

- [x] Checked schema for all modified table for columns that will
auto-update timestamps during migration.
- [x] Confirmed that updating the timestamps is acceptable, and will not
cause unwanted side effects.
- [x] Ensured the correct collation is explicitly set for character
columns (`COLLATE utf8mb4_unicode_ci`).

## fleetd/orbit/Fleet Desktop

- [x] Verified compatibility with the latest released version of Fleet
(see [Must
rule](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/workflows/fleetd-development-and-release-strategy.md))
- [x] Verified that fleetd runs on macOS, Linux and Windows
- [x] Verified auto-update works from the released version of component
to the new version (see [tools/tuf/test](../tools/tuf/test/README.md))


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Configure Orbit to enable debug logging for a limited window on agent
enrollment; enrolled hosts receive debug/verbose behavior while the
window is active and it is reflected in agent config.

* **Chores**
* Added database column to record per-host debug-until timestamps and
datastore support to extend it safely.

* **Tests**
* Added integration and unit tests covering validation, enrollment
stamping, config generation, and runtime debug toggling.

<!-- review_stack_entry_start -->

[![Review Change
Stack](https://storage.googleapis.com/coderabbit_public_assets/review-stack-in-coderabbit-ui.svg)](https://app.coderabbit.ai/change-stack/fleetdm/fleet/pull/45367)

<!-- review_stack_entry_end -->
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Jordan Montgomery
2026-05-14 13:32:02 -04:00
committed by GitHub
co-authored by Copilot Autofix powered by AI
parent cde372df11
commit bee5edaa0b
21 changed files with 804 additions and 12 deletions
@@ -0,0 +1 @@
- Added the `orbit.debug_logging_on_enroll_duration` agent option to allow enabling orbit debug logging for a specified time period after enrollment
@@ -0,0 +1 @@
- Added the `orbit.debug_logging_on_enroll_duration` agent option to allow enabling orbit debug logging for a specified time period after enrollment
+6 -1
View File
@@ -1251,10 +1251,15 @@ func orbitAction(c *cli.Context) error {
orbitClient.RegisterConfigReceiver(luks.New(orbitClient))
}
// Floor for server-driven debug toggling: --debug at startup pins debug on.
startedInDebug := c.Bool("debug")
flagUpdateReceiver := update.NewFlagReceiver(orbitClient.TriggerOrbitRestart, update.FlagUpdateOptions{
RootDir: c.String("root-dir"),
RootDir: c.String("root-dir"),
StartedInDebug: startedInDebug,
})
orbitClient.RegisterConfigReceiver(flagUpdateReceiver)
orbitClient.RegisterConfigReceiver(update.NewDebugLogReceiver(startedInDebug))
if !c.Bool("disable-updates") {
serverOverridesReceiver := newServerOverridesReceiver(
+46
View File
@@ -0,0 +1,46 @@
package update
import (
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
// DebugLogReceiver toggles orbit's zerolog level in response to
// OrbitConfig.DebugLogging.
type DebugLogReceiver struct {
// startedInDebug acts as a floor: when orbit was launched with --debug
// or ORBIT_DEBUG=1, the server can raise the level but cannot lower it.
startedInDebug bool
}
func NewDebugLogReceiver(startedInDebug bool) *DebugLogReceiver {
return &DebugLogReceiver{startedInDebug: startedInDebug}
}
// Run sets the global zerolog level to match config.DebugLogging. Nil
// returns the value to the default, either info level or debug if the
// agent was started in debug mode.
func (r *DebugLogReceiver) Run(config *fleet.OrbitConfig) error {
if config == nil {
return nil
}
currentGlobalLevel := zerolog.GlobalLevel()
desired := zerolog.InfoLevel
if (config.DebugLogging != nil && *config.DebugLogging) || r.startedInDebug {
desired = zerolog.DebugLevel
}
if currentGlobalLevel == desired {
return nil
}
zerolog.SetGlobalLevel(desired)
log.Info().
Str("from", currentGlobalLevel.String()).
Str("to", desired.String()).
Msg("orbit log level changed by server config")
return nil
}
+97
View File
@@ -0,0 +1,97 @@
package update
import (
"testing"
"github.com/fleetdm/fleet/v4/server/fleet"
"github.com/rs/zerolog"
"github.com/stretchr/testify/require"
)
func TestDebugLogReceiver(t *testing.T) {
orig := zerolog.GlobalLevel()
t.Cleanup(func() { zerolog.SetGlobalLevel(orig) })
r := NewDebugLogReceiver(false)
trueVal := true
falseVal := false
cases := []struct {
name string
startLevel zerolog.Level
debugLogging *bool
expectedLevel zerolog.Level
}{
{
name: "true flips info to debug",
startLevel: zerolog.InfoLevel,
debugLogging: &trueVal,
expectedLevel: zerolog.DebugLevel,
},
{
name: "false flips debug to info",
startLevel: zerolog.DebugLevel,
debugLogging: &falseVal,
expectedLevel: zerolog.InfoLevel,
},
{
name: "true is idempotent when already debug",
startLevel: zerolog.DebugLevel,
debugLogging: &trueVal,
expectedLevel: zerolog.DebugLevel,
},
{
name: "false is idempotent when already info",
startLevel: zerolog.InfoLevel,
debugLogging: &falseVal,
expectedLevel: zerolog.InfoLevel,
},
{
name: "nil config field treated as false, idempotent when already info",
startLevel: zerolog.InfoLevel,
debugLogging: nil,
expectedLevel: zerolog.InfoLevel,
},
{
name: "nil config field treated as false, flips debug to info",
startLevel: zerolog.DebugLevel,
debugLogging: nil,
expectedLevel: zerolog.InfoLevel,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
zerolog.SetGlobalLevel(tc.startLevel)
err := r.Run(&fleet.OrbitConfig{DebugLogging: tc.debugLogging})
require.NoError(t, err)
require.Equal(t, tc.expectedLevel, zerolog.GlobalLevel())
})
}
}
func TestDebugLogReceiverStartupFlagIsFloor(t *testing.T) {
orig := zerolog.GlobalLevel()
t.Cleanup(func() { zerolog.SetGlobalLevel(orig) })
r := NewDebugLogReceiver(true)
trueVal := true
falseVal := false
// Server off + already debug: floor keeps it on.
zerolog.SetGlobalLevel(zerolog.DebugLevel)
require.NoError(t, r.Run(&fleet.OrbitConfig{DebugLogging: &falseVal}))
require.Equal(t, zerolog.DebugLevel, zerolog.GlobalLevel())
// Server nil + already debug: floor keeps it on.
zerolog.SetGlobalLevel(zerolog.DebugLevel)
require.NoError(t, r.Run(&fleet.OrbitConfig{DebugLogging: nil}))
require.Equal(t, zerolog.DebugLevel, zerolog.GlobalLevel())
// Server on: always honored.
zerolog.SetGlobalLevel(zerolog.InfoLevel)
require.NoError(t, r.Run(&fleet.OrbitConfig{DebugLogging: &trueVal}))
require.Equal(t, zerolog.DebugLevel, zerolog.GlobalLevel())
}
+24 -6
View File
@@ -30,6 +30,10 @@ type FlagRunner struct {
type FlagUpdateOptions struct {
// RootDir is the root directory for orbit state
RootDir string
// StartedInDebug keeps --verbose and --tls_dump in osquery.flags even
// when the server doesn't push them, so server config can't silently
// override the operator's startup --debug flag.
StartedInDebug bool
}
// NewFlagRunner creates a new runner with provided options
@@ -57,14 +61,28 @@ func (r *FlagRunner) Run(config *fleet.OrbitConfig) error {
flagFileExists = false
}
if len(config.Flags) == 0 {
// command_line_flags not set in YAML, nothing to do
return nil
// Nil/empty Flags is a valid state we must be able to reconcile TO
// (admin cleared command_line_flags, or server-side debug merge turned off).
osqueryFlagMapFromFleet := map[string]string{}
if len(config.Flags) > 0 {
osqueryFlagMapFromFleet, err = getFlagsFromJSON(config.Flags)
if err != nil {
return fmt.Errorf("error parsing flags: %w", err)
}
}
osqueryFlagMapFromFleet, err := getFlagsFromJSON(config.Flags)
if err != nil {
return fmt.Errorf("error parsing flags: %w", err)
// Startup --debug is a floor; admin-specified values still win.
if r.opt.StartedInDebug {
if _, ok := osqueryFlagMapFromFleet["--verbose"]; !ok {
osqueryFlagMapFromFleet["--verbose"] = "true"
}
if _, ok := osqueryFlagMapFromFleet["--tls_dump"]; !ok {
osqueryFlagMapFromFleet["--tls_dump"] = "true"
}
}
if !flagFileExists && len(osqueryFlagMapFromFleet) == 0 {
return nil
}
// compare both flags, if they are equal, nothing to do
+75
View File
@@ -119,3 +119,78 @@ func TestDoFlagsUpdateWithEmptyFlags(t *testing.T) {
require.NoError(t, err)
require.True(t, restartQueued)
}
func TestDoFlagsUpdateWithNilFlags(t *testing.T) {
rootDir := t.TempDir()
osqueryFlagsFile := filepath.Join(rootDir, "osquery.flags")
var restartQueued bool
queueOrbitRestart := func(string) { restartQueued = true }
fr := NewFlagReceiver(queueOrbitRestart, FlagUpdateOptions{RootDir: rootDir})
// Nil Flags + no file: no-op.
err := fr.Run(&fleet.OrbitConfig{Flags: nil})
require.NoError(t, err)
require.False(t, restartQueued)
// Nil Flags + existing file: reconcile to empty + restart.
err = os.WriteFile(osqueryFlagsFile, []byte("--verbose=true\n--tls_dump=true\n"), 0o644)
require.NoError(t, err)
err = fr.Run(&fleet.OrbitConfig{Flags: nil})
require.NoError(t, err)
require.True(t, restartQueued)
contents, err := os.ReadFile(osqueryFlagsFile)
require.NoError(t, err)
require.Empty(t, string(contents))
}
func TestDoFlagsUpdateStartupDebugIsFloor(t *testing.T) {
rootDir := t.TempDir()
osqueryFlagsFile := filepath.Join(rootDir, "osquery.flags")
var restartQueued bool
queueOrbitRestart := func(string) { restartQueued = true }
fr := NewFlagReceiver(queueOrbitRestart, FlagUpdateOptions{
RootDir: rootDir,
StartedInDebug: true,
})
// Nil Flags: floor injects verbose/tls_dump.
err := fr.Run(&fleet.OrbitConfig{Flags: nil})
require.NoError(t, err)
require.True(t, restartQueued)
diskFlags, err := readFlagFile(rootDir)
require.NoError(t, err)
require.Equal(t, "true", diskFlags["--verbose"])
require.Equal(t, "true", diskFlags["--tls_dump"])
// Unrelated flag pushed: floor still preserves verbose/tls_dump.
restartQueued = false
_ = os.Remove(osqueryFlagsFile)
err = fr.Run(&fleet.OrbitConfig{
Flags: json.RawMessage(`{"distributed_interval": 30}`),
})
require.NoError(t, err)
require.True(t, restartQueued)
diskFlags, err = readFlagFile(rootDir)
require.NoError(t, err)
require.Equal(t, "true", diskFlags["--verbose"])
require.Equal(t, "true", diskFlags["--tls_dump"])
require.Equal(t, "30", diskFlags["--distributed_interval"])
// Admin override wins over the floor.
restartQueued = false
err = fr.Run(&fleet.OrbitConfig{
Flags: json.RawMessage(`{"verbose": false}`),
})
require.NoError(t, err)
require.True(t, restartQueued)
diskFlags, err = readFlagFile(rootDir)
require.NoError(t, err)
require.Equal(t, "false", diskFlags["--verbose"])
require.Equal(t, "true", diskFlags["--tls_dump"])
}
+14
View File
@@ -825,6 +825,7 @@ SELECT
h.last_enrolled_at,
h.refetch_requested,
h.refetch_critical_queries_until,
h.orbit_debug_until,
h.team_id,
h.policy_updated_at,
h.public_ip,
@@ -2857,6 +2858,7 @@ func (ds *Datastore) LoadHostByOrbitNodeKey(ctx context.Context, nodeKey string)
h.last_enrolled_at,
h.refetch_requested,
h.refetch_critical_queries_until,
h.orbit_debug_until,
h.team_id,
h.policy_updated_at,
h.public_ip,
@@ -5469,6 +5471,18 @@ func updateHostRefetchRequestedDB(ctx context.Context, tx sqlx.ExtContext, id ui
return ctxerr.Wrapf(ctx, err, "update host %d refetch_requested", id)
}
// ExtendHostOrbitDebugUntil writes `until` only when it is later than the
// current value (or NULL). The conditional WHERE makes the call idempotent
// and prevents lost-update races shortening a longer override.
func (ds *Datastore) ExtendHostOrbitDebugUntil(ctx context.Context, id uint, until time.Time) error {
const stmt = `
UPDATE hosts
SET orbit_debug_until = ?
WHERE id = ? AND (orbit_debug_until IS NULL OR orbit_debug_until < ?)`
_, err := ds.writer(ctx).ExecContext(ctx, stmt, until, id, until)
return ctxerr.Wrapf(ctx, err, "extend host %d orbit_debug_until", id)
}
// UpdateHostRefetchCriticalQueriesUntil updates a host's refetch critical queries until field.
func (ds *Datastore) UpdateHostRefetchCriticalQueriesUntil(ctx context.Context, id uint, until *time.Time) error {
debugLogs := []any{"host_id", id}
+34
View File
@@ -197,6 +197,7 @@ func TestHosts(t *testing.T) {
{"GetHostsLockWipeStatusBatch", testGetHostsLockWipeStatusBatch},
{"HostTimeZone", testHostTimeZone},
{"ListHostsDEPFilters", testListHostsDEPFilters},
{"ExtendHostOrbitDebugUntil", testExtendHostOrbitDebugUntil},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
@@ -13735,3 +13736,36 @@ func testHostsDeleteHostsIdPAccounts(t *testing.T, ds *Datastore) {
require.Equal(t, 0, count, "IdP account should be deleted when Windows host is the last one deleted")
})
}
func testExtendHostOrbitDebugUntil(t *testing.T, ds *Datastore) {
ctx := t.Context()
host := test.NewHost(t, ds, "ehdu", "1.1.1.2", "2", "2", time.Now())
// NULL → value.
first := time.Now().Add(30 * time.Minute).UTC().Truncate(time.Second)
require.NoError(t, ds.ExtendHostOrbitDebugUntil(ctx, host.ID, first))
got, err := ds.Host(ctx, host.ID)
require.NoError(t, err)
require.NotNil(t, got.OrbitDebugUntil)
require.True(t, got.OrbitDebugUntil.Equal(first))
// Earlier → later: extends.
later := first.Add(2 * time.Hour)
require.NoError(t, ds.ExtendHostOrbitDebugUntil(ctx, host.ID, later))
got, err = ds.Host(ctx, host.ID)
require.NoError(t, err)
require.True(t, got.OrbitDebugUntil.Equal(later))
// Later → shorter: no-op.
shorter := first.Add(5 * time.Minute)
require.NoError(t, ds.ExtendHostOrbitDebugUntil(ctx, host.ID, shorter))
got, err = ds.Host(ctx, host.ID)
require.NoError(t, err)
require.True(t, got.OrbitDebugUntil.Equal(later), "extend must not shorten an existing later value")
// Equal: no-op.
require.NoError(t, ds.ExtendHostOrbitDebugUntil(ctx, host.ID, later))
got, err = ds.Host(ctx, host.ID)
require.NoError(t, err)
require.True(t, got.OrbitDebugUntil.Equal(later))
}
@@ -0,0 +1,22 @@
package tables
import (
"database/sql"
"fmt"
)
func init() {
MigrationClient.AddMigration(Up_20260512143542, Down_20260512143542)
}
func Up_20260512143542(tx *sql.Tx) error {
_, err := tx.Exec(`ALTER TABLE hosts ADD COLUMN orbit_debug_until TIMESTAMP(6) NULL DEFAULT NULL`)
if err != nil {
return fmt.Errorf("failed to add orbit_debug_until column to hosts: %w", err)
}
return nil
}
func Down_20260512143542(tx *sql.Tx) error {
return nil
}
@@ -0,0 +1,46 @@
package tables
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestUp_20260512143542(t *testing.T) {
db := applyUpToPrev(t)
_, err := db.Exec(`
INSERT INTO hosts (osquery_host_id, node_key, hostname, uuid)
VALUES (?, ?, ?, ?)`,
"host-1-osquery-id", "host-1-node-key", "host-1", "host-1-uuid",
)
require.NoError(t, err)
applyNext(t, db)
// Default NULL on existing row.
var debugUntil *time.Time
err = db.QueryRow(`SELECT orbit_debug_until FROM hosts WHERE hostname = ?`, "host-1").Scan(&debugUntil)
require.NoError(t, err)
assert.Nil(t, debugUntil)
// Set.
future := time.Now().Add(24 * time.Hour).UTC().Truncate(time.Second)
_, err = db.Exec(`UPDATE hosts SET orbit_debug_until = ? WHERE hostname = ?`, future, "host-1")
require.NoError(t, err)
err = db.QueryRow(`SELECT orbit_debug_until FROM hosts WHERE hostname = ?`, "host-1").Scan(&debugUntil)
require.NoError(t, err)
require.NotNil(t, debugUntil)
assert.True(t, debugUntil.Equal(future), "expected %s, got %s", future, debugUntil)
// Clear.
_, err = db.Exec(`UPDATE hosts SET orbit_debug_until = NULL WHERE hostname = ?`, "host-1")
require.NoError(t, err)
err = db.QueryRow(`SELECT orbit_debug_until FROM hosts WHERE hostname = ?`, "host-1").Scan(&debugUntil)
require.NoError(t, err)
assert.Nil(t, debugUntil)
}
File diff suppressed because one or more lines are too long
+27
View File
@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"strings"
"time"
"github.com/fleetdm/fleet/v4/server/ptr"
)
@@ -28,8 +29,23 @@ type AgentOptions struct {
Extensions json.RawMessage `json:"extensions,omitempty"`
// UpdateChannels holds the configured channels for fleetd components.
UpdateChannels json.RawMessage `json:"update_channels,omitempty"`
// Orbit-agent options. Kept separate from osquery so they bypass the
// osquery schema validator.
Orbit *OrbitAgentOptions `json:"orbit,omitempty"`
}
type OrbitAgentOptions struct {
// DebugLoggingOnEnrollDuration is the number of seconds (0 to
// MaxOrbitDebugLoggingOnEnrollDurationSeconds) that every host enrolling
// under this scope is stamped with orbit_debug_until = now() + duration.
DebugLoggingOnEnrollDuration int `json:"debug_logging_on_enroll_duration"`
}
const (
MaxOrbitDebugLoggingOnEnrollDurationSeconds = 24 * 60 * 60 // 86400 = 24h
MaxOrbitDebugLoggingOnEnrollDuration = MaxOrbitDebugLoggingOnEnrollDurationSeconds * time.Second
)
type AgentOptionsOverrides struct {
// Platforms is a map from platform name to the config override.
Platforms map[string]json.RawMessage `json:"platforms,omitempty"`
@@ -115,6 +131,17 @@ func ValidateJSONAgentOptions(ctx context.Context, ds Datastore, rawJSON json.Ra
}
}
if opts.Orbit != nil {
if s := opts.Orbit.DebugLoggingOnEnrollDuration; s != 0 {
if s < 0 {
return errors.New("orbit.debug_logging_on_enroll_duration must not be negative")
}
if s > MaxOrbitDebugLoggingOnEnrollDurationSeconds {
return fmt.Errorf("orbit.debug_logging_on_enroll_duration must not exceed %d seconds", MaxOrbitDebugLoggingOnEnrollDurationSeconds)
}
}
}
if len(opts.Config) > 0 {
if err := validateJSONAgentOptionsSet(opts.Config); err != nil {
return fmt.Errorf("common config: %w", err)
+9 -1
View File
@@ -196,8 +196,16 @@ func TestValidateAgentOptions(t *testing.T) {
},
"command_line_flags": {
"logger_tls_backoff_max": 200
}
}
}`, true, ``},
{"orbit debug_logging_on_enroll_duration valid", `{"orbit": {"debug_logging_on_enroll_duration": 3600}}`, true, ``},
{"orbit debug_logging_on_enroll_duration zero", `{"orbit": {"debug_logging_on_enroll_duration": 0}}`, true, ``},
{"orbit debug_logging_on_enroll_duration max", `{"orbit": {"debug_logging_on_enroll_duration": 86400}}`, true, ``},
{"orbit debug_logging_on_enroll_duration over max", `{"orbit": {"debug_logging_on_enroll_duration": 86401}}`, true, `must not exceed 86400 seconds`},
{"orbit debug_logging_on_enroll_duration negative", `{"orbit": {"debug_logging_on_enroll_duration": -1}}`, true, `must not be negative`},
{"orbit debug_logging_on_enroll_duration string rejected", `{"orbit": {"debug_logging_on_enroll_duration": "1h"}}`, true, `cannot unmarshal string`},
{"orbit unknown subkey rejected", `{"orbit": {"foo": true}}`, true, `unknown field "foo"`},
}
for _, c := range cases {
+4
View File
@@ -1101,6 +1101,10 @@ type Datastore interface {
// UpdateHostRefetchCriticalQueriesUntil updates a host's refetch critical queries until field.
UpdateHostRefetchCriticalQueriesUntil(ctx context.Context, hostID uint, until *time.Time) error
// ExtendHostOrbitDebugUntil writes `until` only when it is later than the
// current value (or NULL): idempotent, never shortens an existing override.
ExtendHostOrbitDebugUntil(ctx context.Context, hostID uint, until time.Time) error
// FlippingPoliciesForHost fetches the policies with incoming results and returns:
// - a list of "new" failing policies; "new" here means those that fail on their first
// run, and those that were passing on the previous run and are failing on the incoming execution.
+4
View File
@@ -397,6 +397,10 @@ type Host struct {
// so we don't need this.
RefetchCriticalQueriesUntil *time.Time `json:"refetch_critical_queries_until" db:"refetch_critical_queries_until" csv:"-"` //nolint:apiparamcheck
// When non-nil and in the future, the orbit config response sets
// debug_logging=true until that time.
OrbitDebugUntil *time.Time `json:"orbit_debug_until,omitempty" db:"orbit_debug_until" csv:"-"`
// DEPAssignedToFleet is set to true if the host is assigned to Fleet in Apple Business.
// It is a *bool becase we want it to be returned from only a subset of endpoints related to
// Orbit and Fleet Desktop. Otherwise, it will be set to NULL so it is omitted from JSON
+2
View File
@@ -67,6 +67,8 @@ type OrbitConfig struct {
//
// If UpdateChannels is nil it means the server isn't using/setting this feature.
UpdateChannels *OrbitUpdateChannels `json:"update_channels,omitempty"`
// nil = no opinion (orbit keeps its current level); true/false sets it.
DebugLogging *bool `json:"debug_logging,omitempty"`
}
type OrbitConfigReceiver interface {
+12
View File
@@ -795,6 +795,8 @@ type UpdateHostRefetchRequestedFunc func(ctx context.Context, hostID uint, value
type UpdateHostRefetchCriticalQueriesUntilFunc func(ctx context.Context, hostID uint, until *time.Time) error
type ExtendHostOrbitDebugUntilFunc func(ctx context.Context, hostID uint, until time.Time) error
type FlippingPoliciesForHostFunc func(ctx context.Context, hostID uint, incomingResults map[uint]*bool) (newFailing []uint, newPassing []uint, err error)
type RecordPolicyQueryExecutionsFunc func(ctx context.Context, host *fleet.Host, results map[uint]*bool, updated time.Time, deferredSaveHost bool, newlyPassingPolicyIDs []uint) error
@@ -3128,6 +3130,9 @@ type DataStore struct {
UpdateHostRefetchCriticalQueriesUntilFunc UpdateHostRefetchCriticalQueriesUntilFunc
UpdateHostRefetchCriticalQueriesUntilFuncInvoked bool
ExtendHostOrbitDebugUntilFunc ExtendHostOrbitDebugUntilFunc
ExtendHostOrbitDebugUntilFuncInvoked bool
FlippingPoliciesForHostFunc FlippingPoliciesForHostFunc
FlippingPoliciesForHostFuncInvoked bool
@@ -7594,6 +7599,13 @@ func (s *DataStore) UpdateHostRefetchCriticalQueriesUntil(ctx context.Context, h
return s.UpdateHostRefetchCriticalQueriesUntilFunc(ctx, hostID, until)
}
func (s *DataStore) ExtendHostOrbitDebugUntil(ctx context.Context, hostID uint, until time.Time) error {
s.mu.Lock()
s.ExtendHostOrbitDebugUntilFuncInvoked = true
s.mu.Unlock()
return s.ExtendHostOrbitDebugUntilFunc(ctx, hostID, until)
}
func (s *DataStore) FlippingPoliciesForHost(ctx context.Context, hostID uint, incomingResults map[uint]*bool) (newFailing []uint, newPassing []uint, err error) {
s.mu.Lock()
s.FlippingPoliciesForHostFuncInvoked = true
+70
View File
@@ -17135,3 +17135,73 @@ func (s *integrationTestSuite) TestOrgLogoUpload() {
}
assert.True(t, sawAutoCleanupActivity, "auto-cleanup must emit a deleted_org_logo activity for the affected mode")
}
func (s *integrationTestSuite) TestOrbitDebugLoggingOnEnroll() {
t := s.T()
ctx := context.Background()
// Reject above cap.
var acResp appConfigResponse
s.DoJSON("PATCH", "/api/latest/fleet/config", json.RawMessage(`{
"agent_options": { "orbit": {"debug_logging_on_enroll_duration": 86401} }
}`), http.StatusBadRequest, &acResp)
// Reject negative.
s.DoJSON("PATCH", "/api/latest/fleet/config", json.RawMessage(`{
"agent_options": { "orbit": {"debug_logging_on_enroll_duration": -1} }
}`), http.StatusBadRequest, &acResp)
// Reject duration string (must be seconds, integer).
s.DoJSON("PATCH", "/api/latest/fleet/config", json.RawMessage(`{
"agent_options": { "orbit": {"debug_logging_on_enroll_duration": "1h"} }
}`), http.StatusBadRequest, &acResp)
// 1h global window (3600 seconds).
s.DoJSON("PATCH", "/api/latest/fleet/config", json.RawMessage(`{
"agent_options": { "orbit": {"debug_logging_on_enroll_duration": 3600} }
}`), http.StatusOK, &acResp)
secret := uuid.New().String()
var applyResp applyEnrollSecretSpecResponse
s.DoJSON("POST", "/api/latest/fleet/spec/enroll_secret", applyEnrollSecretSpecRequest{
Spec: &fleet.EnrollSecretSpec{
Secrets: []*fleet.EnrollSecret{{Secret: secret}},
},
}, http.StatusOK, &applyResp)
beforeEnroll := time.Now()
var enrollResp enrollOrbitResponse
hostUUID := uuid.New().String()
s.DoJSON("POST", "/api/fleet/orbit/enroll", fleet.EnrollOrbitRequest{
EnrollSecret: secret,
HardwareUUID: hostUUID,
HardwareSerial: uuid.New().String(),
Hostname: "enroll-debug-stamped",
Platform: "linux",
}, http.StatusOK, &enrollResp)
require.NotEmpty(t, enrollResp.OrbitNodeKey)
stampedHost, err := s.ds.LoadHostByOrbitNodeKey(ctx, enrollResp.OrbitNodeKey)
require.NoError(t, err)
require.NotNil(t, stampedHost.OrbitDebugUntil)
require.WithinDuration(t, beforeEnroll.Add(time.Hour), *stampedHost.OrbitDebugUntil, time.Minute)
// Clearing the option stops stamping.
s.DoJSON("PATCH", "/api/latest/fleet/config", json.RawMessage(`{
"agent_options": { "orbit": {"debug_logging_on_enroll_duration": 0} }
}`), http.StatusOK, &acResp)
var enrollResp2 enrollOrbitResponse
hostUUID2 := uuid.New().String()
s.DoJSON("POST", "/api/fleet/orbit/enroll", fleet.EnrollOrbitRequest{
EnrollSecret: secret,
HardwareUUID: hostUUID2,
HardwareSerial: uuid.New().String(),
Hostname: "enroll-debug-not-stamped",
Platform: "linux",
}, http.StatusOK, &enrollResp2)
unstampedHost, err := s.ds.LoadHostByOrbitNodeKey(ctx, enrollResp2.OrbitNodeKey)
require.NoError(t, err)
require.Nil(t, unstampedHost.OrbitDebugUntil)
}
+102 -2
View File
@@ -9,6 +9,7 @@ import (
"log/slog"
"net/http"
"net/url"
"time"
"github.com/fleetdm/fleet/v4/ee/server/service/hostidentity/httpsig"
"github.com/fleetdm/fleet/v4/server"
@@ -327,9 +328,69 @@ func (svc *Service) EnrollOrbit(ctx context.Context, hostInfo fleet.OrbitHostInf
svc.logger.ErrorContext(ctx, "record fleet enroll activity", "err", err)
}
// Non-fatal: enrollment must succeed even if the debug stamp fails.
if err := svc.maybeStampOrbitDebugFromAgentOptions(ctx, host, appConfig); err != nil {
svc.logger.ErrorContext(ctx, "failed to stamp orbit debug from agent options on enroll", "err", err)
}
return orbitNodeKey, nil
}
// maybeStampOrbitDebugFromAgentOptions stamps orbit_debug_until = now()+duration
// when the host's effective agent options set debug_logging_on_enroll_duration.
func (svc *Service) maybeStampOrbitDebugFromAgentOptions(ctx context.Context, host *fleet.Host, appConfig *fleet.AppConfig) error {
rawOpts, err := svc.loadAgentOptionsForHost(ctx, host, appConfig)
if err != nil {
return ctxerr.Wrap(ctx, err, "load agent options for enroll debug stamp")
}
if len(rawOpts) == 0 {
return nil
}
var opts fleet.AgentOptions
if err := json.Unmarshal(rawOpts, &opts); err != nil {
return ctxerr.Wrap(ctx, err, "unmarshal agent options for enroll debug stamp")
}
if opts.Orbit == nil || opts.Orbit.DebugLoggingOnEnrollDuration <= 0 {
return nil
}
// Defense in depth: validator already caps on write.
seconds := min(opts.Orbit.DebugLoggingOnEnrollDuration, fleet.MaxOrbitDebugLoggingOnEnrollDurationSeconds)
duration := time.Duration(seconds) * time.Second
until := svc.clock.Now().Add(duration).UTC().Truncate(time.Second)
if err := svc.ds.ExtendHostOrbitDebugUntil(ctx, host.ID, until); err != nil {
return ctxerr.Wrap(ctx, err, "set orbit_debug_until on enroll")
}
svc.logger.InfoContext(ctx, "stamped orbit debug logging on enroll",
"host_id", host.ID,
"team_id", host.TeamID,
"orbit_debug_until", until,
"duration", duration.String(),
)
return nil
}
// loadAgentOptionsForHost returns the team's agent_options if the host is in
// a team, otherwise the global options from AppConfig.
func (svc *Service) loadAgentOptionsForHost(ctx context.Context, host *fleet.Host, appConfig *fleet.AppConfig) (json.RawMessage, error) {
if host.TeamID != nil {
opts, err := svc.ds.TeamAgentOptions(ctx, *host.TeamID)
if err != nil {
return nil, err
}
if opts == nil {
return nil, nil
}
return *opts, nil
}
if appConfig.AgentOptions == nil {
return nil, nil
}
return *appConfig.AgentOptions, nil
}
func getOrbitConfigEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (fleet.Errorer, error) {
cfg, err := svc.GetOrbitConfig(ctx)
if err != nil {
@@ -338,6 +399,33 @@ func getOrbitConfigEndpoint(ctx context.Context, request interface{}, svc fleet.
return fleet.OrbitGetConfigResponse{OrbitConfig: cfg}, nil
}
// resolveOrbitDebugLogging returns the merged command_line_flags and the
// *bool to set on OrbitConfig.DebugLogging. When the host's orbit_debug_until
// is unset or expired, returns (flags unchanged, nil). When active, merges
// verbose=true into the flags map without clobbering admin-specified values.
func resolveOrbitDebugLogging(ctx context.Context, host *fleet.Host, flags json.RawMessage) (json.RawMessage, *bool, error) {
if host == nil || host.OrbitDebugUntil == nil || !host.OrbitDebugUntil.After(time.Now()) {
return flags, nil, nil
}
adminFlags := map[string]any{}
if len(flags) > 0 {
if err := json.Unmarshal(flags, &adminFlags); err != nil {
return nil, nil, ctxerr.Wrap(ctx, err, "orbit debug logging: parse command_line_flags")
}
}
if _, ok := adminFlags["verbose"]; !ok {
adminFlags["verbose"] = true
}
merged, err := json.Marshal(adminFlags)
if err != nil {
return nil, nil, ctxerr.Wrap(ctx, err, "orbit debug logging: marshal merged flags")
}
debug := true
return merged, &debug, nil
}
func (svc *Service) GetOrbitConfig(ctx context.Context) (fleet.OrbitConfig, error) {
// this is not a user-authenticated endpoint
svc.authz.SkipAuthorization(ctx)
@@ -576,13 +664,19 @@ func (svc *Service) GetOrbitConfig(ctx context.Context) (fleet.OrbitConfig, erro
_ = svc.ds.ClearPendingEscrow(ctx, host.ID)
}
mergedFlags, debugLogging, err := resolveOrbitDebugLogging(ctx, host, opts.CommandLineStartUpFlags)
if err != nil {
return fleet.OrbitConfig{}, err
}
return fleet.OrbitConfig{
ScriptExeTimeout: opts.ScriptExecutionTimeout,
Flags: opts.CommandLineStartUpFlags,
Flags: mergedFlags,
Extensions: extensionsFiltered,
Notifications: notifs,
NudgeConfig: nudgeConfig,
UpdateChannels: updateChannels,
DebugLogging: debugLogging,
}, nil
}
@@ -651,13 +745,19 @@ func (svc *Service) GetOrbitConfig(ctx context.Context) (fleet.OrbitConfig, erro
_ = svc.ds.ClearPendingEscrow(ctx, host.ID)
}
mergedFlags, debugLogging, err := resolveOrbitDebugLogging(ctx, host, opts.CommandLineStartUpFlags)
if err != nil {
return fleet.OrbitConfig{}, err
}
return fleet.OrbitConfig{
ScriptExeTimeout: opts.ScriptExecutionTimeout,
Flags: opts.CommandLineStartUpFlags,
Flags: mergedFlags,
Extensions: extensionsFiltered,
Notifications: notifs,
NudgeConfig: nudgeConfig,
UpdateChannels: updateChannels,
DebugLogging: debugLogging,
}, nil
}
+205
View File
@@ -1210,3 +1210,208 @@ type orbitTestNotFoundErr struct{}
func (e *orbitTestNotFoundErr) Error() string { return "not found" }
func (e *orbitTestNotFoundErr) IsNotFound() bool { return true }
func rawJSON(s string) *json.RawMessage {
r := json.RawMessage(s)
return &r
}
func TestMaybeStampOrbitDebugFromAgentOptions(t *testing.T) {
getInternal := func(svc fleet.Service) *Service {
return ((svc.(validationMiddleware)).Service).(*Service)
}
t.Run("no agent options -> no stamp", func(t *testing.T) {
ds := new(mock.Store)
svc, ctx := newTestService(t, ds, nil, nil, &TestServerOpts{SkipCreateTestUsers: true})
host := &fleet.Host{ID: 1}
appCfg := &fleet.AppConfig{}
err := getInternal(svc).maybeStampOrbitDebugFromAgentOptions(ctx, host, appCfg)
require.NoError(t, err)
require.False(t, ds.ExtendHostOrbitDebugUntilFuncInvoked)
})
t.Run("zero duration -> no stamp", func(t *testing.T) {
ds := new(mock.Store)
svc, ctx := newTestService(t, ds, nil, nil, &TestServerOpts{SkipCreateTestUsers: true})
host := &fleet.Host{ID: 1}
appCfg := &fleet.AppConfig{
AgentOptions: rawJSON(`{"orbit": {"debug_logging_on_enroll_duration": 0}}`),
}
err := getInternal(svc).maybeStampOrbitDebugFromAgentOptions(ctx, host, appCfg)
require.NoError(t, err)
require.False(t, ds.ExtendHostOrbitDebugUntilFuncInvoked)
})
t.Run("global option set, no team -> stamps from app config", func(t *testing.T) {
ds := new(mock.Store)
svc, ctx := newTestService(t, ds, nil, nil, &TestServerOpts{SkipCreateTestUsers: true})
host := &fleet.Host{ID: 42}
appCfg := &fleet.AppConfig{
AgentOptions: rawJSON(`{"orbit": {"debug_logging_on_enroll_duration": 3600}}`),
}
var gotID uint
var gotUntil time.Time
ds.ExtendHostOrbitDebugUntilFunc = func(ctx context.Context, hostID uint, until time.Time) error {
gotID = hostID
gotUntil = until
return nil
}
before := time.Now()
err := getInternal(svc).maybeStampOrbitDebugFromAgentOptions(ctx, host, appCfg)
require.NoError(t, err)
require.True(t, ds.ExtendHostOrbitDebugUntilFuncInvoked)
require.Equal(t, host.ID, gotID)
require.WithinDuration(t, before.Add(time.Hour), gotUntil, time.Minute)
})
t.Run("team option set -> stamps from team agent options, ignores global", func(t *testing.T) {
ds := new(mock.Store)
svc, ctx := newTestService(t, ds, nil, nil, &TestServerOpts{SkipCreateTestUsers: true})
teamID := uint(7)
host := &fleet.Host{ID: 99, TeamID: &teamID}
appCfg := &fleet.AppConfig{
// Team membership: team options win, global is ignored.
AgentOptions: rawJSON(`{"orbit": {"debug_logging_on_enroll_duration": 86400}}`),
}
ds.TeamAgentOptionsFunc = func(ctx context.Context, id uint) (*json.RawMessage, error) {
require.Equal(t, teamID, id)
return rawJSON(`{"orbit": {"debug_logging_on_enroll_duration": 1800}}`), nil
}
var gotUntil time.Time
ds.ExtendHostOrbitDebugUntilFunc = func(ctx context.Context, hostID uint, until time.Time) error {
gotUntil = until
return nil
}
before := time.Now()
err := getInternal(svc).maybeStampOrbitDebugFromAgentOptions(ctx, host, appCfg)
require.NoError(t, err)
require.True(t, ds.TeamAgentOptionsFuncInvoked)
require.True(t, ds.ExtendHostOrbitDebugUntilFuncInvoked)
require.WithinDuration(t, before.Add(30*time.Minute), gotUntil, time.Minute)
})
t.Run("team has no agent options row -> no stamp, no fallback to global", func(t *testing.T) {
ds := new(mock.Store)
svc, ctx := newTestService(t, ds, nil, nil, &TestServerOpts{SkipCreateTestUsers: true})
teamID := uint(7)
host := &fleet.Host{ID: 99, TeamID: &teamID}
appCfg := &fleet.AppConfig{
AgentOptions: rawJSON(`{"orbit": {"debug_logging_on_enroll_duration": 3600}}`),
}
ds.TeamAgentOptionsFunc = func(ctx context.Context, id uint) (*json.RawMessage, error) {
return nil, nil
}
err := getInternal(svc).maybeStampOrbitDebugFromAgentOptions(ctx, host, appCfg)
require.NoError(t, err)
require.True(t, ds.TeamAgentOptionsFuncInvoked)
require.False(t, ds.ExtendHostOrbitDebugUntilFuncInvoked)
})
t.Run("over-cap value defensively clamped at 24h", func(t *testing.T) {
ds := new(mock.Store)
svc, ctx := newTestService(t, ds, nil, nil, &TestServerOpts{SkipCreateTestUsers: true})
host := &fleet.Host{ID: 1}
// Bypass the validator by stuffing a too-large value directly.
appCfg := &fleet.AppConfig{
AgentOptions: rawJSON(`{"orbit": {"debug_logging_on_enroll_duration": 360000}}`),
}
var gotUntil time.Time
ds.ExtendHostOrbitDebugUntilFunc = func(ctx context.Context, hostID uint, until time.Time) error {
gotUntil = until
return nil
}
before := time.Now()
err := getInternal(svc).maybeStampOrbitDebugFromAgentOptions(ctx, host, appCfg)
require.NoError(t, err)
require.WithinDuration(t, before.Add(fleet.MaxOrbitDebugLoggingOnEnrollDuration), gotUntil, time.Minute)
})
}
func TestResolveOrbitDebugLogging(t *testing.T) {
ctx := t.Context()
future := time.Now().Add(time.Hour)
past := time.Now().Add(-time.Hour)
cases := []struct {
name string
host *fleet.Host
inFlags json.RawMessage
wantDebug *bool
wantFlags map[string]any
}{
{
name: "no host -> nil debug, flags unchanged",
host: nil,
inFlags: nil,
wantDebug: nil,
},
{
name: "no override -> nil debug, flags unchanged",
host: &fleet.Host{},
inFlags: json.RawMessage(`{"distributed_interval":10}`),
wantDebug: nil,
},
{
name: "unexpired override -> debug on, flags merged",
host: &fleet.Host{OrbitDebugUntil: &future},
inFlags: nil,
wantDebug: new(true),
wantFlags: map[string]any{
"verbose": true,
},
},
{
name: "unexpired override with admin flags -> merged",
host: &fleet.Host{OrbitDebugUntil: &future},
inFlags: json.RawMessage(`{"distributed_interval":10}`),
wantDebug: new(true),
wantFlags: map[string]any{
"distributed_interval": float64(10),
"verbose": true,
},
},
{
name: "admin verbose:false wins over debug-on",
host: &fleet.Host{OrbitDebugUntil: &future},
inFlags: json.RawMessage(`{"verbose":false}`),
wantDebug: new(true),
wantFlags: map[string]any{
"verbose": false,
},
},
{
name: "expired override is ignored",
host: &fleet.Host{OrbitDebugUntil: &past},
inFlags: nil,
wantDebug: nil,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
gotFlags, gotDebug, err := resolveOrbitDebugLogging(ctx, tc.host, tc.inFlags)
require.NoError(t, err)
if tc.wantDebug == nil {
require.Nil(t, gotDebug)
require.Equal(t, tc.inFlags, gotFlags)
return
}
require.NotNil(t, gotDebug)
require.Equal(t, *tc.wantDebug, *gotDebug)
var got map[string]any
require.NoError(t, json.Unmarshal(gotFlags, &got))
require.Equal(t, tc.wantFlags, got)
})
}
}