Discharge7443/fleet
1
0
Fork 0
Code Issues Pull Requests Actions 19 Packages Projects Releases Wiki Activity

Update fleetctl convert for schedulable queries (#13125)

Browse Source
This commit is contained in:
Jacob Shandling
2023-08-08 10:39:20 -07:00
committed by GitHub
parent 6555d8def4
commit a7f1501054
5 changed files with 563 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
changes/12657-update-fleetctl-convert
+1
View File
@@ -0,0 +1 @@
- Update `fleetctl convert` to convert packs to the new combined schedule and query format
cmd/fleetctl/convert.go
+79 -47
View File
@@ -19,15 +19,60 @@ import (
"github.com/urfave/cli/v2"
)
// mappings based on https://github.com/osquery/osquery/blob/b87a4b5f1567415a72acd5ecd0e9e7ab75754959/tools/codegen/genwebsitejson.py#L38C18-L38C18
var platformMapping = map[string][]string{
"darwin": {"darwin"},
"linux": {"linux"},
"windows": {"windows"},
"chrome": {"chrome"},
"specs": {"darwin", "linux", "windows"},
"utility": {"darwin", "linux", "windows"},
"yara": {"darwin", "linux", "windows"},
"smart": {"darwin", "linux"},
"kernel": {"darwin"},
"linwin": {"linux", "windows"},
"macwin": {"darwin", "windows"},
"posix": {"darwin", "linux"},
"sleuthkit": {"darwin", "linux"},
"any": {""},
"all": {""},
"": {""},
}
func convertPlatforms(platformsIn string) (string, error) {
splitPlatformsIn := strings.Split(platformsIn, ",")
// validate and convert each substring
mapped := map[string]struct{}{} // use a set to dedupe
for _, substring := range splitPlatformsIn {
mappedSubstring, ok := platformMapping[substring]
// validate substring
if !ok {
return "", fmt.Errorf("unsupported platform: %s", substring)
}
for _, p := range mappedSubstring {
mapped[p] = struct{}{}
}
}
// convert set to slice
result := make([]string, 0, len(mapped))
for p := range mapped {
result = append(result, p)
}
// sort for deterministic output
sort.Strings(result)
resultString := strings.Join(result, ",")
return resultString, nil
}
func specGroupFromPack(name string, inputPack fleet.PermissivePackContent) (*spec.Group, error) {
specs := &spec.Group{
Queries: []*fleet.QuerySpec{},
Packs: []*fleet.PackSpec{},
Labels: []*fleet.LabelSpec{},
}
pack := &fleet.PackSpec{
Name: name,
}
// this ensures order is consistent in output
@@ -41,12 +86,8 @@ func specGroupFromPack(name string, inputPack fleet.PermissivePackContent) (*spe
for _, name := range keys {
query := inputPack.Queries[name]
spec := &fleet.QuerySpec{
Name: name,
Description: query.Description,
Query: query.Query,
}
// get the interval as uint from a variety of possible types
interval := uint(0)
switch i := query.Interval.(type) {
case string:
@@ -61,21 +102,33 @@ func specGroupFromPack(name string, inputPack fleet.PermissivePackContent) (*spe
interval = uint(i)
}
specs.Queries = append(specs.Queries, spec)
pack.Queries = append(pack.Queries, fleet.PackSpecQuery{
Name: name,
QueryName: name,
Interval: interval,
Description: query.Description,
Snapshot: query.Snapshot,
Removed: query.Removed,
Shard: query.Shard,
Platform: query.Platform,
Version: query.Version,
})
}
// handle nil query.Platform
var queryPlatforms string
if query.Platform != nil {
queryPlatforms = *query.Platform
}
convertedPlatforms, err := convertPlatforms(queryPlatforms)
if err != nil {
return nil, err
}
specs.Packs = append(specs.Packs, pack)
// handle nil query.Version
var minOsqueryVersion string
if query.Version != nil {
minOsqueryVersion = *query.Version
}
spec := &fleet.QuerySpec{
Name: name,
Description: query.Description,
Query: query.Query,
Interval: interval,
Platform: convertedPlatforms,
MinOsqueryVersion: minOsqueryVersion,
}
specs.Queries = append(specs.Queries, spec)
}
return specs, nil
}
@@ -87,7 +140,7 @@ func convertCommand() *cli.Command {
)
return &cli.Command{
Name: "convert",
Usage: "Convert osquery packs into decomposed fleet configs",
Usage: "Convert osquery packs into Fleet queries",
UsageText: `fleetctl convert [options]`,
Flags: []cli.Flag{
configFlag(),
@@ -151,27 +204,6 @@ func convertCommand() *cli.Command {
w = file
}
for _, pack := range specs.Packs {
specBytes, err := json.Marshal(pack)
if err != nil {
return err
}
meta := spec.Metadata{
Kind: fleet.PackKind,
Version: fleet.ApiVersion,
Spec: specBytes,
}
out, err := yaml.Marshal(meta)
if err != nil {
return err
}
fmt.Fprintln(w, "---")
fmt.Fprint(w, string(out))
}
for _, query := range specs.Queries {
specBytes, err := json.Marshal(query)
if err != nil {
cmd/fleetctl/convert_test.go
+1 -1
View File
@@ -64,5 +64,5 @@ func TestConvertFileStdout(t *testing.T) {
os.Stdout = oldStdout
w.Close()
out, _ := ioutil.ReadAll(r)
require.Equal(t, string(expected), string(out))
require.YAMLEq(t, string(expected), string(out))
}
cmd/fleetctl/testdata/convert_input.conf
Vendored
+180 -30
View File
@@ -1,44 +1,194 @@
{
"queries": {
"launchd": {
"query" : "select * from launchd;",
"interval" : "3600",
"platform" : "darwin",
"version" : "1.4.5",
"description" : "Retrieves all the daemons that will run in the start of the target OSX system.",
"value" : "Identify malware that uses this persistence mechanism to launch at system boot"
"query": "select * from launchd;",
"interval": "3600",
"platform": "darwin",
"version": "1.4.5",
"description": "Retrieves all the daemons that will run in the start of the target OSX system.",
"value": "Identify malware that uses this persistence mechanism to launch at system boot"
},
"disk_encryption": {
"query" : "select * from disk_encryption;",
"interval" : "86400",
"disk_encryption (posix)": {
"query": "select * from disk_encryption;",
"interval": "86400",
"platform": "posix",
"version" : "1.4.5",
"description" : "Retrieves the current disk encryption status for the target system.",
"value" : "Identifies a system potentially vulnerable to disk cloning."
"version": "1.4.5",
"description": "Retrieves the current disk encryption status for the target system.",
"value": "Identifies a system potentially vulnerable to disk cloning."
},
"disk_encryption (darwin,linux)": {
"query": "select * from disk_encryption;",
"interval": "300",
"platform": "darwin,linux",
"version": "1.4.5",
"description": "Retrieves the current disk encryption status for the target system.",
"value": "Identifies a system potentially vulnerable to disk cloning."
},
"iptables": {
"query" : "select * from iptables;",
"interval" : "3600",
"platform" : "linux",
"version" : "1.4.5",
"description" : "Retrieves the current filters and chains per filter in the target system.",
"value" : "Verify firewall settings are as restrictive as you need. Identify unwanted firewall holes made by malware or humans"
"query": "select * from iptables;",
"interval": "3600",
"platform": "linux",
"version": "1.4.5",
"description": "Retrieves the current filters and chains per filter in the target system.",
"value": "Verify firewall settings are as restrictive as you need. Identify unwanted firewall holes made by malware or humans"
},
"app_schemes": {
"query" : "select * from app_schemes;",
"interval" : "86400",
"platform" : "darwin",
"version" : "1.4.7",
"description" : "Retrieves the list of application scheme/protocol-based IPC handlers.",
"value" : "Post-priori hijack detection, detect potential sensitive information leakage."
"query": "select * from app_schemes;",
"interval": "86400",
"platform": "darwin",
"version": "1.4.7",
"description": "Retrieves the list of application scheme/protocol-based IPC handlers.",
"value": "Post-priori hijack detection, detect potential sensitive information leakage."
},
"sandboxes": {
"query" : "select * from sandboxes;",
"interval" : "86400",
"platform" : "darwin",
"version" : "1.4.7",
"description" : "Lists the application bundle that owns a sandbox label.",
"value" : "Post-priori hijack detection, detect potential sensitive information leakage."
"query": "select * from sandboxes;",
"interval": "86400",
"platform": "darwin",
"version": "1.4.7",
"description": "Lists the application bundle that owns a sandbox label.",
"value": "Post-priori hijack detection, detect potential sensitive information leakage."
},
"disk_info": {
"query": "select * from disk_info;",
"interval": "86400",
"platform": "chrome,windows",
"version": "1.4.7",
"description": "Retrieve basic information about the physical disks of a system.",
"value": "Identify scary possibilities with disks."
},
"listening_ports (specs)": {
"query": "select * from listening_ports;",
"interval": "3600",
"platform": "specs",
"version": "1.4.7",
"description": "Retrieves the list of listening ports.",
"value": "Identify unwanted open ports."
},
"listening_ports (utility)": {
"query": "select * from listening_ports;",
"interval": "3600",
"platform": "utility",
"version": "1.4.7",
"description": "Retrieves the list of listening ports.",
"value": "Identify unwanted open ports."
},
"yara (yara)": {
"query": "select * from yara;",
"interval": "0",
"platform": "yara",
"version": "1.4.7",
"description": "Triggers one-off YARA query for files at the specified path. Requires one of sig_group, sigfile, or sigrule.",
"value": "TBD"
},
"ulimit_info (smart)": {
"query": "select * from ulimit_info;",
"interval": "300",
"platform": "smart",
"version": "1.4.7",
"description": "System resource usage limits.",
"value": "Identify potential resource exhaustion attacks."
},
"uptime (kernel)": {
"query": "select * from uptime;",
"interval": "600",
"platform": "kernel",
"version": "1.4.7",
"description": "System uptime.",
"value": "Identify systems that have been rebooted recently."
},
"uptime (linwin)": {
"query": "select * from uptime;",
"interval": "600",
"platform": "linwin",
"version": "1.4.7",
"description": "System uptime.",
"value": "Identify systems that have been rebooted recently."
},
"uptime (macwin)": {
"query": "select * from uptime;",
"interval": "600",
"platform": "macwin",
"version": "1.4.7",
"description": "System uptime.",
"value": "Identify systems that have been rebooted recently."
},
"uptime (sleuthkit)": {
"query": "select * from uptime;",
"interval": "600",
"platform": "sleuthkit",
"version": "1.4.7",
"description": "System uptime.",
"value": "Identify systems that have been rebooted recently."
},
"windows crashes": {
"query": "select * from windows_crashes;",
"interval": "3600",
"platform": "windows",
"version": "1.4.7",
"description": "Extracted information from Windows crash logs (Minidumps).",
"value": "Identify systems that have been rebooted recently."
},
"user groups (any)": {
"query": "select * from user_groups;",
"interval": "3600",
"platform": "any",
"version": "1.4.7",
"description": "List of all user groups.",
"value": "Identify unwanted user groups."
},
"user groups (missing platform)": {
"query": "select * from user_groups;",
"interval": "3600",
"version": "1.4.7",
"description": "List of all user groups.",
"value": "Identify unwanted user groups."
},
"user groups (missing version)": {
"query": "select * from user_groups;",
"interval": "3600",
"platform": "darwin",
"description": "List of all user groups.",
"value": "Identify unwanted user groups."
},
"user groups (all)": {
"query": "select * from user_groups;",
"interval": "3600",
"platform": "all",
"version": "1.4.7",
"description": "List of all user groups.",
"value": "Identify unwanted user groups."
},
"user groups (empty string platform, empty string version)": {
"query": "select * from user_groups;",
"interval": "3600",
"platform": "",
"version": "",
"description": "List of all user groups.",
"value": "Identify unwanted user groups."
},
"user groups (darwin,linux)": {
"query": "select * from user_groups;",
"interval": "3600",
"platform": "darwin,linux",
"version": "1.4.7",
"description": "List of all user groups.",
"value": "Identify unwanted user groups."
},
"user groups (linux,darwin)": {
"query": "select * from user_groups;",
"interval": "3600",
"platform": "linux,darwin",
"version": "1.4.7",
"description": "List of all user groups.",
"value": "Identify unwanted user groups."
},
"user groups (windows,chrome)": {
"query": "select * from user_groups;",
"interval": "3600",
"platform": "windows,chrome",
"version": "1.4.7",
"description": "List of all user groups.",
"value": "Identify unwanted user groups."
}
}
}
cmd/fleetctl/testdata/convert_output.yml
Vendored
+302 -61
View File
@@ -1,57 +1,15 @@
---
apiVersion: v1
kind: pack
spec:
disabled: false
name: convert_input
queries:
- description: Retrieves the list of application scheme/protocol-based IPC handlers.
interval: 86400
name: app_schemes
platform: darwin
query: app_schemes
version: 1.4.7
- description: Retrieves the current disk encryption status for the target system.
interval: 86400
name: disk_encryption
platform: posix
query: disk_encryption
version: 1.4.5
- description: Retrieves the current filters and chains per filter in the target
system.
interval: 3600
name: iptables
platform: linux
query: iptables
version: 1.4.5
- description: Retrieves all the daemons that will run in the start of the target
OSX system.
interval: 3600
name: launchd
platform: darwin
query: launchd
version: 1.4.5
- description: Lists the application bundle that owns a sandbox label.
interval: 86400
name: sandboxes
platform: darwin
query: sandboxes
version: 1.4.7
targets:
labels: null
teams: null
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: Retrieves the list of application scheme/protocol-based IPC handlers.
interval: 0
interval: 86400
logging: ""
min_osquery_version: ""
min_osquery_version: 1.4.7
name: app_schemes
observer_can_run: false
platform: ""
platform: darwin
query: select * from app_schemes;
team: ""
---
@@ -60,12 +18,12 @@ kind: query
spec:
automations_enabled: false
description: Retrieves the current disk encryption status for the target system.
interval: 0
interval: 86400
logging: ""
min_osquery_version: ""
name: disk_encryption
min_osquery_version: 1.4.5
name: disk_encryption (posix)
observer_can_run: false
platform: ""
platform: darwin,linux
query: select * from disk_encryption;
team: ""
---
@@ -73,13 +31,41 @@ apiVersion: v1
kind: query
spec:
automations_enabled: false
description: Retrieves the current filters and chains per filter in the target system.
interval: 0
description: Retrieves the current disk encryption status for the target system.
interval: 300
logging: ""
min_osquery_version: ""
min_osquery_version: 1.4.5
name: disk_encryption (darwin,linux)
observer_can_run: false
platform: darwin,linux
query: select * from disk_encryption;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: Retrieve basic information about the physical disks of a system.
interval: 86400
logging: ""
min_osquery_version: 1.4.7
name: disk_info
observer_can_run: false
platform: chrome,windows
query: select * from disk_info;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: Retrieves the current filters and chains per filter in the target system.
interval: 3600
logging: ""
min_osquery_version: 1.4.5
name: iptables
observer_can_run: false
platform: ""
platform: linux
query: select * from iptables;
team: ""
---
@@ -87,14 +73,15 @@ apiVersion: v1
kind: query
spec:
automations_enabled: false
description: Retrieves all the daemons that will run in the start of the target
description:
Retrieves all the daemons that will run in the start of the target
OSX system.
interval: 0
interval: 3600
logging: ""
min_osquery_version: ""
min_osquery_version: 1.4.5
name: launchd
observer_can_run: false
platform: ""
platform: darwin
query: select * from launchd;
team: ""
---
@@ -102,12 +89,266 @@ apiVersion: v1
kind: query
spec:
automations_enabled: false
description: Lists the application bundle that owns a sandbox label.
interval: 0
description: Retrieves the list of listening ports.
interval: 3600
logging: ""
min_osquery_version: ""
min_osquery_version: 1.4.7
name: listening_ports (specs)
observer_can_run: false
platform: darwin,linux,windows
query: select * from listening_ports;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: Retrieves the list of listening ports.
interval: 3600
logging: ""
min_osquery_version: 1.4.7
name: listening_ports (utility)
observer_can_run: false
platform: darwin,linux,windows
query: select * from listening_ports;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: Lists the application bundle that owns a sandbox label.
interval: 86400
logging: ""
min_osquery_version: 1.4.7
name: sandboxes
observer_can_run: false
platform: ""
platform: darwin
query: select * from sandboxes;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: System resource usage limits.
interval: 300
logging: ""
min_osquery_version: 1.4.7
name: ulimit_info (smart)
observer_can_run: false
platform: darwin,linux
query: select * from ulimit_info;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: System uptime.
interval: 600
logging: ""
min_osquery_version: 1.4.7
name: uptime (kernel)
observer_can_run: false
platform: darwin
query: select * from uptime;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: System uptime.
interval: 600
logging: ""
min_osquery_version: 1.4.7
name: uptime (linwin)
observer_can_run: false
platform: linux,windows
query: select * from uptime;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: System uptime.
interval: 600
logging: ""
min_osquery_version: 1.4.7
name: uptime (macwin)
observer_can_run: false
platform: darwin,windows
query: select * from uptime;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: System uptime.
interval: 600
logging: ""
min_osquery_version: 1.4.7
name: uptime (sleuthkit)
observer_can_run: false
platform: darwin,linux
query: select * from uptime;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: Lists the application bundle that owns a sandbox label.
interval: 86400
logging: ""
min_osquery_version: 1.4.7
name: sandboxes
observer_can_run: false
platform: darwin
query: select * from sandboxes;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: List of all user groups.
interval: 3600
logging: ""
min_osquery_version: 1.4.7
name: user groups (all)
observer_can_run: false
platform: ""
query: select * from user_groups;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: List of all user groups.
interval: 3600
logging: ""
min_osquery_version: 1.4.7
name: user groups (any)
observer_can_run: false
platform: ""
query: select * from user_groups;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: List of all user groups.
interval: 3600
logging: ""
min_osquery_version: 1.4.7
name: user groups (darwin,linux)
observer_can_run: false
platform: darwin,linux
query: select * from user_groups;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: List of all user groups.
interval: 3600
logging: ""
min_osquery_version: ""
name: user groups (empty string platform, empty string version)
observer_can_run: false
platform: ""
query: select * from user_groups;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: List of all user groups.
interval: 3600
logging: ""
min_osquery_version: 1.4.7
name: user groups (linux,darwin)
observer_can_run: false
platform: darwin,linux
query: select * from user_groups;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: List of all user groups.
interval: 3600
logging: ""
min_osquery_version: 1.4.7
name: user groups (missing platform)
observer_can_run: false
platform: ""
query: select * from user_groups;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: List of all user groups.
interval: 3600
logging: ""
min_osquery_version: ""
name: user groups (missing version)
observer_can_run: false
platform: darwin
query: select * from user_groups;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: List of all user groups.
interval: 3600
logging: ""
min_osquery_version: 1.4.7
name: user groups (windows,chrome)
observer_can_run: false
platform: chrome,windows
query: select * from user_groups;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description: Extracted information from Windows crash logs (Minidumps).
interval: 3600
logging: ""
min_osquery_version: 1.4.7
name: windows crashes
observer_can_run: false
platform: windows
query: select * from windows_crashes;
team: ""
---
apiVersion: v1
kind: query
spec:
automations_enabled: false
description:
Triggers one-off YARA query for files at the specified path. Requires
one of sig_group, sigfile, or sigrule.
interval: 0
logging: ""
min_osquery_version: 1.4.7
name: yara (yara)
observer_can_run: false
platform: darwin,linux,windows
query: select * from yara;
team: ""