Add capability to serve YARA rules via authenticated Fleet endpoints (#23343)

Implements the Fleet side of #14899

- Add new endpoints to update and retrieve yara rules
- Add support in fleetctl for applying the rules

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

- [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`.
  See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/Committing-Changes.md#changes-files) for more information.
- [x] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements)
- [ ] Added support on fleet's osquery simulator `cmd/osquery-perf` for new osquery data ingestion features.
- [x] Added/updated tests
- [ ] If paths of existing endpoints are modified without backwards compatibility, checked the frontend/CLI for any necessary changes
- [x] If database migrations are included, checked table schema to confirm autoupdate
- For database migrations:
  - [x] Checked schema for all modified table for columns that will auto-update timestamps during migration.
  - [ ] Confirmed that updating the timestamps is acceptable, and will not cause unwanted side effects.
  - [x] Ensured the correct collation is explicitly set for character columns (`COLLATE utf8mb4_unicode_ci`).
- [x] Manual QA for all new/changed functionality
This commit is contained in:
Zach Wasserman
2024-11-13 09:01:08 -08:00
committed by GitHub
parent 552e76b68e
commit 8c21dff636
18 changed files with 381 additions and 8 deletions
+1
View File
@@ -0,0 +1 @@
* Added capability for Fleet to serve yara rules to agents over HTTPS authenticated via node key (requires osquery 5.14+).
+5 -2
View File
@@ -137,8 +137,7 @@ func runServerWithMockedDS(t *testing.T, opts ...*service.TestServerOpts) (*http
fleet.MDMAssetCAKey: "scepkey",
}, nil
}
ds.GetAllMDMConfigAssetsByNameFunc = func(ctx context.Context, assetNames []fleet.MDMAssetName,
_ sqlx.QueryerContext) (map[fleet.MDMAssetName]fleet.MDMConfigAsset, error) {
ds.GetAllMDMConfigAssetsByNameFunc = func(ctx context.Context, assetNames []fleet.MDMAssetName, _ sqlx.QueryerContext) (map[fleet.MDMAssetName]fleet.MDMConfigAsset, error) {
return map[fleet.MDMAssetName]fleet.MDMConfigAsset{
fleet.MDMAssetABMCert: {Name: fleet.MDMAssetABMCert, Value: certPEM},
fleet.MDMAssetABMKey: {Name: fleet.MDMAssetABMKey, Value: keyPEM},
@@ -150,6 +149,10 @@ func runServerWithMockedDS(t *testing.T, opts ...*service.TestServerOpts) (*http
}, nil
}
ds.ApplyYaraRulesFunc = func(context.Context, []fleet.YaraRule) error {
return nil
}
var cachedDS fleet.Datastore
if len(opts) > 0 && opts[0].NoCacheDatastore {
cachedDS = ds
+49
View File
@@ -288,3 +288,52 @@ func (ds *Datastore) getConfigEnableDiskEncryption(ctx context.Context, teamID *
}
return ac.MDM.EnableDiskEncryption.Value, nil
}
func (ds *Datastore) ApplyYaraRules(ctx context.Context, rules []fleet.YaraRule) error {
return ds.withRetryTxx(ctx, func(tx sqlx.ExtContext) error {
return applyYaraRulesDB(ctx, tx, rules)
})
}
func applyYaraRulesDB(ctx context.Context, q sqlx.ExtContext, rules []fleet.YaraRule) error {
const delStmt = "DELETE FROM yara_rules"
if _, err := q.ExecContext(ctx, delStmt); err != nil {
return ctxerr.Wrap(ctx, err, "clear before insert")
}
if len(rules) > 0 {
const insStmt = `INSERT INTO yara_rules (name, contents) VALUES %s`
var args []interface{}
sql := fmt.Sprintf(insStmt, strings.TrimSuffix(strings.Repeat(`(?, ?),`, len(rules)), ","))
for _, r := range rules {
args = append(args, r.Name, r.Contents)
}
if _, err := q.ExecContext(ctx, sql, args...); err != nil {
return ctxerr.Wrap(ctx, err, "insert yara rules")
}
}
return nil
}
func (ds *Datastore) GetYaraRules(ctx context.Context) ([]fleet.YaraRule, error) {
sql := "SELECT name, contents FROM yara_rules"
rules := []fleet.YaraRule{}
if err := sqlx.SelectContext(ctx, ds.reader(ctx), &rules, sql); err != nil {
return nil, ctxerr.Wrap(ctx, err, "get yara rules")
}
return rules, nil
}
func (ds *Datastore) YaraRuleByName(ctx context.Context, name string) (*fleet.YaraRule, error) {
query := "SELECT name, contents FROM yara_rules WHERE name = ?"
rule := fleet.YaraRule{}
if err := sqlx.GetContext(ctx, ds.reader(ctx), &rule, query, name); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, ctxerr.Wrap(ctx, notFound("YaraRule"), "no yara rule with provided name")
}
return nil, ctxerr.Wrap(ctx, err, "get yara rule by name")
}
return &rule, nil
}
+101 -1
View File
@@ -37,6 +37,7 @@ func TestAppConfig(t *testing.T) {
{"GetConfigEnableDiskEncryption", testGetConfigEnableDiskEncryption},
{"IsEnrollSecretAvailable", testIsEnrollSecretAvailable},
{"NDESSCEPProxyPassword", testNDESSCEPProxyPassword},
{"YaraRulesRoundtrip", testYaraRulesRoundtrip},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
@@ -533,7 +534,6 @@ func testIsEnrollSecretAvailable(t *testing.T, ds *Datastore) {
},
)
}
}
func testNDESSCEPProxyPassword(t *testing.T, ds *Datastore) {
@@ -606,5 +606,105 @@ func testNDESSCEPProxyPassword(t *testing.T, ds *Datastore) {
require.NoError(t, err)
checkProxyConfig()
checkPassword()
}
func testYaraRulesRoundtrip(t *testing.T, ds *Datastore) {
ctx := context.Background()
defer TruncateTables(t, ds)
// Empty insert
expectedRules := []fleet.YaraRule{}
err := ds.ApplyYaraRules(ctx, expectedRules)
require.NoError(t, err)
rules, err := ds.GetYaraRules(ctx)
require.NoError(t, err)
assert.Equal(t, expectedRules, rules)
// Insert values
expectedRules = []fleet.YaraRule{
{
Name: "wildcard.yar",
Contents: `rule WildcardExample
{
strings:
$hex_string = { E2 34 ?? C8 A? FB }
condition:
$hex_string
}`,
},
{
Name: "jump.yar",
Contents: `rule JumpExample
{
strings:
$hex_string = { F4 23 [4-6] 62 B4 }
condition:
$hex_string
}`,
},
}
err = ds.ApplyYaraRules(ctx, expectedRules)
require.NoError(t, err)
rules, err = ds.GetYaraRules(ctx)
require.NoError(t, err)
assert.Equal(t, expectedRules, rules)
rule, err := ds.YaraRuleByName(ctx, expectedRules[0].Name)
require.NoError(t, err)
assert.Equal(t, &expectedRules[0], rule)
rule, err = ds.YaraRuleByName(ctx, expectedRules[1].Name)
require.NoError(t, err)
assert.Equal(t, &expectedRules[1], rule)
// Update rules
expectedRules = []fleet.YaraRule{
{
Name: "wildcard.yar",
Contents: `rule WildcardExample
{
strings:
$hex_string = { E2 34 ?? C8 A? FB }
condition:
$hex_string
}`,
},
{
Name: "jump-modified.yar",
Contents: `rule JumpExample
{
strings:
$hex_string = true
condition:
$hex_string
}`,
},
}
err = ds.ApplyYaraRules(ctx, expectedRules)
require.NoError(t, err)
rules, err = ds.GetYaraRules(ctx)
require.NoError(t, err)
assert.Equal(t, expectedRules, rules)
rule, err = ds.YaraRuleByName(ctx, expectedRules[0].Name)
require.NoError(t, err)
assert.Equal(t, &expectedRules[0], rule)
rule, err = ds.YaraRuleByName(ctx, expectedRules[1].Name)
require.NoError(t, err)
assert.Equal(t, &expectedRules[1], rule)
// Clear rules
expectedRules = []fleet.YaraRule{}
err = ds.ApplyYaraRules(ctx, expectedRules)
require.NoError(t, err)
rules, err = ds.GetYaraRules(ctx)
require.NoError(t, err)
assert.Equal(t, expectedRules, rules)
// Get rule that doesn't exist
_, err = ds.YaraRuleByName(ctx, "wildcard.yar")
require.Error(t, err)
}
@@ -0,0 +1,29 @@
package tables
import (
"database/sql"
"fmt"
)
func init() {
MigrationClient.AddMigration(Up_20241016155452, Down_20241016155452)
}
func Up_20241016155452(tx *sql.Tx) error {
_, err := tx.Exec(`
CREATE TABLE yara_rules (
id INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,
name VARCHAR(255) NOT NULL,
contents MEDIUMTEXT NOT NULL,
PRIMARY KEY (id),
UNIQUE KEY idx_yara_rules_name (name)
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;`)
if err != nil {
return fmt.Errorf("failed to create yara_rules table: %w", err)
}
return nil
}
func Down_20241016155452(tx *sql.Tx) error {
return nil
}
File diff suppressed because one or more lines are too long
+3 -1
View File
@@ -1,4 +1,4 @@
// Automatically generated by tools/osquery-agent-options for osquery 5.12.2. DO NOT EDIT!
// Automatically generated by tools/osquery-agent-options for osquery 5.14.1. DO NOT EDIT!
// To update flags for a new osquery version, update the osqueryVersion variable in
// "tools/osquery-agent-options/main.go" and run "cd server/fleet/ && go generate".
package fleet
@@ -115,6 +115,7 @@ type osqueryOptions struct {
TlsDisableStatusLog bool `json:"tls_disable_status_log"`
Verbose bool `json:"verbose"`
YaraDelay uint32 `json:"yara_delay"`
YaraSigurlAuthenticate bool `json:"yara_sigurl_authenticate"`
// embed the os-specific structs
OsqueryCommandLineFlagsLinux
@@ -297,6 +298,7 @@ type osqueryCommandLineFlags struct {
WatchdogMemoryLimit uint64 `json:"watchdog_memory_limit"`
WatchdogUtilizationLimit uint64 `json:"watchdog_utilization_limit"`
YaraDelay uint32 `json:"yara_delay"`
YaraSigurlAuthenticate bool `json:"yara_sigurl_authenticate"`
// embed the os-specific structs
OsqueryCommandLineFlagsLinux
+17
View File
@@ -518,6 +518,8 @@ type AppConfig struct {
// (The source of truth for scripts is in MySQL.)
Scripts optjson.Slice[string] `json:"scripts"`
YaraRules []YaraRule `json:"yara_rules,omitempty"`
// when true, strictDecoding causes the UnmarshalJSON method to return an
// error if there are unknown fields in the raw JSON.
strictDecoding bool
@@ -683,6 +685,12 @@ func (c *AppConfig) Copy() *AppConfig {
clone.MDM.MacOSSetup.Software = optjson.SetSlice(sw)
}
if c.YaraRules != nil {
rules := make([]YaraRule, len(c.YaraRules))
copy(rules, c.YaraRules)
clone.YaraRules = rules
}
return &clone
}
@@ -1377,3 +1385,12 @@ type WindowsSettings struct {
// (The source of truth for profiles is in MySQL.)
CustomSettings optjson.Slice[MDMProfileSpec] `json:"custom_settings"`
}
type YaraRuleSpec struct {
Path string `json:"path"`
}
type YaraRule struct {
Name string `json:"name"`
Contents string `json:"contents"`
}
+5
View File
@@ -439,6 +439,11 @@ type Datastore interface {
// value.
AggregateEnrollSecretPerTeam(ctx context.Context) ([]*EnrollSecret, error)
// Methods for getting and applying the stored yara rules.
GetYaraRules(ctx context.Context) ([]YaraRule, error)
ApplyYaraRules(ctx context.Context, rules []YaraRule) error
YaraRuleByName(ctx context.Context, name string) (*YaraRule, error)
///////////////////////////////////////////////////////////////////////////////
// InviteStore contains the methods for managing user invites in a datastore.
+1
View File
@@ -66,6 +66,7 @@ type OsqueryService interface {
) (err error)
SubmitStatusLogs(ctx context.Context, logs []json.RawMessage) (err error)
SubmitResultLogs(ctx context.Context, logs []json.RawMessage) (err error)
YaraRuleByName(ctx context.Context, name string) (*YaraRule, error)
}
type Service interface {
+36
View File
@@ -327,6 +327,12 @@ type ApplyEnrollSecretsFunc func(ctx context.Context, teamID *uint, secrets []*f
type AggregateEnrollSecretPerTeamFunc func(ctx context.Context) ([]*fleet.EnrollSecret, error)
type GetYaraRulesFunc func(ctx context.Context) ([]fleet.YaraRule, error)
type ApplyYaraRulesFunc func(ctx context.Context, rules []fleet.YaraRule) error
type YaraRuleByNameFunc func(ctx context.Context, name string) (*fleet.YaraRule, error)
type NewInviteFunc func(ctx context.Context, i *fleet.Invite) (*fleet.Invite, error)
type ListInvitesFunc func(ctx context.Context, opt fleet.ListOptions) ([]*fleet.Invite, error)
@@ -1601,6 +1607,15 @@ type DataStore struct {
AggregateEnrollSecretPerTeamFunc AggregateEnrollSecretPerTeamFunc
AggregateEnrollSecretPerTeamFuncInvoked bool
GetYaraRulesFunc GetYaraRulesFunc
GetYaraRulesFuncInvoked bool
ApplyYaraRulesFunc ApplyYaraRulesFunc
ApplyYaraRulesFuncInvoked bool
YaraRuleByNameFunc YaraRuleByNameFunc
YaraRuleByNameFuncInvoked bool
NewInviteFunc NewInviteFunc
NewInviteFuncInvoked bool
@@ -3896,6 +3911,27 @@ func (s *DataStore) AggregateEnrollSecretPerTeam(ctx context.Context) ([]*fleet.
return s.AggregateEnrollSecretPerTeamFunc(ctx)
}
func (s *DataStore) GetYaraRules(ctx context.Context) ([]fleet.YaraRule, error) {
s.mu.Lock()
s.GetYaraRulesFuncInvoked = true
s.mu.Unlock()
return s.GetYaraRulesFunc(ctx)
}
func (s *DataStore) ApplyYaraRules(ctx context.Context, rules []fleet.YaraRule) error {
s.mu.Lock()
s.ApplyYaraRulesFuncInvoked = true
s.mu.Unlock()
return s.ApplyYaraRulesFunc(ctx, rules)
}
func (s *DataStore) YaraRuleByName(ctx context.Context, name string) (*fleet.YaraRule, error) {
s.mu.Lock()
s.YaraRuleByNameFuncInvoked = true
s.mu.Unlock()
return s.YaraRuleByNameFunc(ctx, name)
}
func (s *DataStore) NewInvite(ctx context.Context, i *fleet.Invite) (*fleet.Invite, error) {
s.mu.Lock()
s.NewInviteFuncInvoked = true
+6
View File
@@ -773,6 +773,12 @@ func (svc *Service) ModifyAppConfig(ctx context.Context, p []byte, applyOpts fle
return nil, ctxerr.Wrap(ctx, err, "process iPadOS OS updates config change")
}
if appConfig.YaraRules != nil {
if err := svc.ds.ApplyYaraRules(ctx, appConfig.YaraRules); err != nil {
return nil, ctxerr.Wrap(ctx, err, "save yara rules for app config")
}
}
// if the Windows updates requirements changed, create the corresponding
// activity.
if !oldAppConfig.MDM.WindowsUpdates.Equal(appConfig.MDM.WindowsUpdates) {
+62 -1
View File
@@ -524,7 +524,7 @@ func (c *Client) ApplyGroup(
for i, f := range scripts {
b, err := os.ReadFile(f)
if err != nil {
return nil, nil, nil, fmt.Errorf("applying fleet config: %w", err)
return nil, nil, nil, fmt.Errorf("applying no-team scripts: %w", err)
}
scriptPayloads[i] = fleet.ScriptPayload{
ScriptContents: b,
@@ -537,6 +537,27 @@ func (c *Client) ApplyGroup(
}
teamsScripts["No team"] = noTeamScripts
}
rules, err := extractAppCfgYaraRules(specs.AppConfig)
if err != nil {
return nil, nil, nil, fmt.Errorf("applying yara rules: %w", err)
}
if rules != nil {
rulePayloads := make([]fleet.YaraRule, len(rules))
for i, f := range rules {
path := resolveApplyRelativePath(baseDir, f.Path)
b, err := os.ReadFile(path)
if err != nil {
return nil, nil, nil, fmt.Errorf("applying yara rules: %w", err)
}
rulePayloads[i] = fleet.YaraRule{
Contents: string(b),
Name: filepath.Base(f.Path),
}
}
specs.AppConfig.(map[string]interface{})["yara_rules"] = rulePayloads
}
if err := c.ApplyAppConfig(specs.AppConfig, opts.ApplySpecOptions); err != nil {
return nil, nil, nil, fmt.Errorf("applying fleet config: %w", err)
}
@@ -1137,6 +1158,46 @@ func extractAppCfgScripts(appCfg interface{}) []string {
return scriptsStrings
}
func extractAppCfgYaraRules(appCfg interface{}) ([]fleet.YaraRuleSpec, error) {
asMap, ok := appCfg.(map[string]interface{})
if !ok {
return nil, errors.New("extract yara rules: app config is not a map")
}
rules, ok := asMap["yara_rules"]
if !ok {
// yara_rules is not present. Return an empty slice so that the value is cleared.
return []fleet.YaraRuleSpec{}, nil
}
rulesAny, ok := rules.([]interface{})
if !ok || rulesAny == nil {
// If nil, return an empty slice so the value will be cleared.
return []fleet.YaraRuleSpec{}, nil
}
ruleSpecs := make([]fleet.YaraRuleSpec, 0, len(rulesAny))
for _, v := range rulesAny {
smap, ok := v.(map[string]interface{})
if !ok {
return nil, errors.New("extract yara rules: rule entry is not a map")
}
pathEntry, ok := smap["path"]
if !ok {
return nil, errors.New("extract yara rules: rule entry missing path")
}
path, ok := pathEntry.(string)
if !ok {
return nil, errors.New("extract yara rules: rule entry path is not string")
}
ruleSpecs = append(ruleSpecs, fleet.YaraRuleSpec{Path: path})
}
return ruleSpecs, nil
}
type profileSpecsByPlatform struct {
macos []fleet.MDMProfileSpec
windows []fleet.MDMProfileSpec
+2
View File
@@ -857,6 +857,8 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC
POST("/api/osquery/carve/begin", carveBeginEndpoint, carveBeginRequest{})
he.WithAltPaths("/api/v1/osquery/log").
POST("/api/osquery/log", submitLogsEndpoint, submitLogsRequest{})
he.WithAltPaths("/api/v1/osquery/yara/{name}").
POST("/api/osquery/yara/{name}", getYaraEndpoint, getYaraRequest{})
// orbit authenticated endpoints
oe := newOrbitAuthenticatedEndpointer(svc, logger, opts, r, apiVersions...)
+12
View File
@@ -26,6 +26,8 @@ type SubmitStatusLogsFunc func(ctx context.Context, logs []json.RawMessage) (err
type SubmitResultLogsFunc func(ctx context.Context, logs []json.RawMessage) (err error)
type YaraRuleByNameFunc func(ctx context.Context, name string) (*fleet.YaraRule, error)
type TLSService struct {
EnrollAgentFunc EnrollAgentFunc
EnrollAgentFuncInvoked bool
@@ -48,6 +50,9 @@ type TLSService struct {
SubmitResultLogsFunc SubmitResultLogsFunc
SubmitResultLogsFuncInvoked bool
YaraRuleByNameFunc YaraRuleByNameFunc
YaraRuleByNameFuncInvoked bool
mu sync.Mutex
}
@@ -99,3 +104,10 @@ func (s *TLSService) SubmitResultLogs(ctx context.Context, logs []json.RawMessag
s.mu.Unlock()
return s.SubmitResultLogsFunc(ctx, logs)
}
func (s *TLSService) YaraRuleByName(ctx context.Context, name string) (*fleet.YaraRule, error) {
s.mu.Lock()
s.YaraRuleByNameFuncInvoked = true
s.mu.Unlock()
return s.YaraRuleByNameFunc(ctx, name)
}
+36
View File
@@ -2488,3 +2488,39 @@ func getQueryNameAndTeamIDFromResult(path string) (*uint, string, error) {
// If none of the above patterns match, return error
return nil, "", fmt.Errorf("unknown format: %q", path)
}
// Yara rules
func (svc *Service) YaraRuleByName(ctx context.Context, name string) (*fleet.YaraRule, error) {
return svc.ds.YaraRuleByName(ctx, name)
}
type getYaraRequest struct {
NodeKey string `json:"node_key"`
Name string `url:"name"`
}
func (r *getYaraRequest) hostNodeKey() string {
return r.NodeKey
}
type getYaraResponse struct {
Err error `json:"error,omitempty"`
Content string
}
func (r getYaraResponse) error() error { return r.Err }
func (r getYaraResponse) hijackRender(ctx context.Context, w http.ResponseWriter) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
_, _ = w.Write([]byte(r.Content))
}
func getYaraEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (errorer, error) {
r := request.(*getYaraRequest)
rule, err := svc.YaraRuleByName(ctx, r.Name)
if err != nil {
return getYaraResponse{Err: err}, nil
}
return getYaraResponse{Content: rule.Contents}, nil
}
@@ -176,5 +176,8 @@ github.com/fleetdm/fleet/v4/server/fleet/AppConfig Scripts optjson.Slice[string]
github.com/fleetdm/fleet/v4/pkg/optjson/Slice[string] Set bool
github.com/fleetdm/fleet/v4/pkg/optjson/Slice[string] Valid bool
github.com/fleetdm/fleet/v4/pkg/optjson/Slice[string] Value []string
github.com/fleetdm/fleet/v4/server/fleet/AppConfig YaraRules []fleet.YaraRule
github.com/fleetdm/fleet/v4/server/fleet/YaraRule Name string
github.com/fleetdm/fleet/v4/server/fleet/YaraRule Contents string
github.com/fleetdm/fleet/v4/server/fleet/AppConfig strictDecoding bool
github.com/fleetdm/fleet/v4/server/fleet/AppConfig didUnmarshalLegacySettings []string
+1 -1
View File
@@ -27,7 +27,7 @@ import (
var (
rxOption = regexp.MustCompile(`\-\-(\w+)\s`)
osqueryVersion = "5.12.2"
osqueryVersion = "5.14.1"
structTpl = template.Must(template.New("struct").Funcs(template.FuncMap{
"camelCase": camelCaseOptionName,